def get_attribute_value(cls, attributes, attribute_name, first_only=False): """ The attribute value type must be decodable (str in py2, bytes in py3) :type attributes: dict :type attribute_name: unicode :type first_only: bool """ attribute_values = attributes.get(attribute_name) if attribute_values: try: if first_only or len(attribute_values) == 1: return attribute_values[0].decode(cls.encoding) else: return [ val.decode(cls.encoding) for val in attribute_values ] except UnicodeError as e: raise AssertionException( "Encoding error in value of attribute '%s': %s" % (attribute_name, e)) return None
def iter_search_result(self, filter_string, attributes): """ type: filter_string: str type: attributes: list(str) """ attr_dict = OKTAValueFormatter.get_extended_attribute_dict(attributes) try: self.logger.info( "Calling okta SDK get_users with the following %s", filter_string) if attr_dict: users = self.users_client.get_all_users( query=filter_string, extended_attribute=attr_dict) else: users = self.users_client.get_all_users(query=filter_string) except OktaError as e: self.logger.warning("Unable to query users") raise AssertionException("Okta error querying for users: %s" % e) return users
def run(self, post_sync_data): """ Run the Sign sync :param post_sync_data: :return: """ if self.test_mode: self.logger.info("Sign Sync disabled in test mode") return for org_name, sign_client in self.clients.items(): # create any new Sign groups for new_group in set(self.user_groups[org_name]) - set( sign_client.sign_groups()): self.logger.info( "Creating new Sign group: {}".format(new_group)) sign_client.create_group(new_group) umapi_users = post_sync_data.umapi_data.get(org_name) if umapi_users is None: raise AssertionException( "Error getting umapi users from post_sync_data") self.update_sign_users(umapi_users, sign_client, org_name)
def get_attribute_value(cls, attributes, attribute_name, first_only=False): """ The attribute value type must be decodable (str in py2, bytes in py3) :type attributes: dict :type attribute_name: unicode :type first_only: bool """ attribute_values = attributes.get(attribute_name) if attribute_values: try: if isinstance(attribute_values, six.string_types): return attribute_values else: if first_only: return attribute_values[0] return attribute_values except UnicodeError as e: raise AssertionException( "Encoding error in value of attribute '%s': %s" % (attribute_name, e)) return None
def iter_group_members(self, group, filter_string, extended_attributes): """ :type group: str :type filter_string: str :type extended_attributes: list :rtype iterator(str, str) """ user_attribute_names = [ "firstName", "lastName", "login", "email", "countryCode" ] extended_attributes = list( set(extended_attributes) - set(user_attribute_names)) user_attribute_names.extend(extended_attributes) res_group = self.find_group(group) if res_group: try: attr_dict = OKTAValueFormatter.get_extended_attribute_dict( user_attribute_names) members = self.groups_client.get_group_all_users( res_group.id, attr_dict) except OktaError as e: self.logger.warning("Unable to get_group_users") raise AssertionException( "Okta error querying for group users: %s" % e) # Filtering users based all_users_filter query in config for member in self.filter_users(members, filter_string): profile = member.profile if not profile.email: self.logger.warning('No email attribute for login: %s', profile.login) continue user = self.convert_user(member, extended_attributes) if not user: continue yield (user) else: self.logger.warning("No group found for: %s", group)
def __init__(self, caller_options): ''' :type caller_options: dict ''' self.options = options = { # these are in alphabetical order! Always add new ones that way! 'delete_strays': False, 'config_file_encoding': 'ascii', 'directory_connector_module_name': None, 'directory_connector_overridden_options': None, 'directory_group_filter': None, 'directory_group_mapped': False, 'disentitle_strays': False, 'exclude_strays': False, 'main_config_filename': DEFAULT_MAIN_CONFIG_FILENAME, 'manage_groups': False, 'remove_strays': False, 'stray_list_input_path': None, 'stray_list_output_path': None, 'test_mode': False, 'update_user_info': True, 'username_filter_regex': None, } options.update(caller_options) main_config_filename = options.get('main_config_filename') config_encoding = options['config_file_encoding'] try: codecs.lookup(config_encoding) except LookupError: raise AssertionException( "Unknown encoding '%s' specified with --config-file-encoding" % config_encoding) ConfigFileLoader.config_encoding = config_encoding main_config_content = ConfigFileLoader.load_root_config( main_config_filename) self.logger = logger = logging.getLogger('config') logger.info("Using main config file: %s", main_config_filename) self.main_config = DictConfig("<%s>" % main_config_filename, main_config_content)
def get_options(caller_config): builder = user_sync.config.OptionsBuilder(caller_config) builder.set_string_value('group_filter_format', six.text_type( '(&(|(objectCategory=group)(objectClass=groupOfNames)(objectClass=posixGroup))(cn={group}))')) builder.set_string_value('all_users_filter', six.text_type( '(&(objectClass=user)(objectCategory=person)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))')) builder.set_string_value('group_member_filter_format', None) builder.set_bool_value('require_tls_cert', False) builder.set_dict_value('two_steps_lookup', None) builder.set_string_value('string_encoding', 'utf8') builder.set_string_value('user_identity_type_format', None) builder.set_string_value('user_email_format', six.text_type('{mail}')) builder.set_string_value('user_username_format', None) builder.set_string_value('user_domain_format', None) builder.set_string_value('user_given_name_format', six.text_type('{givenName}')) builder.set_string_value('user_surname_format', six.text_type('{sn}')) builder.set_string_value('user_country_code_format', six.text_type('{c}')) builder.set_string_value('user_identity_type', None) builder.set_int_value('search_page_size', 200) builder.set_string_value('logger_name', LDAPDirectoryConnector.name) builder.require_string_value('host') builder.require_string_value('username') builder.require_string_value('base_dn') options = builder.get_options() options['two_steps_enabled'] = False if options['two_steps_lookup'] is not None: ts_config = caller_config.get_dict_config('two_steps_lookup', True) ts_builder = user_sync.config.OptionsBuilder(ts_config) ts_builder.require_string_value('group_member_attribute_name') ts_builder.set_bool_value('nested_group', False) options['two_steps_enabled'] = True options['two_steps_lookup'] = ts_builder.get_options() if options['group_member_filter_format']: raise AssertionException( "Cannot define both 'group_member_attribute_name' and 'group_member_filter_format' in config") else: if not options['group_member_filter_format']: options['group_member_filter_format'] = six.text_type('(memberOf={group_dn})') return options
def load_users_and_groups(self, groups, extended_attributes, all_users): """ :type groups: list(str) :type extended_attributes: list(str) :type all_users: bool :rtype (bool, iterable(dict)) """ if all_users: raise AssertionException( "Okta connector has no notion of all users, please specify a --users group" ) options = self.options all_users_filter = options['all_users_filter'] self.logger.info('Loading users...') self.user_by_uid = user_by_uid = {} for group in groups: total_group_members = 0 total_group_users = 0 for user in self.iter_group_members(group, all_users_filter, extended_attributes): total_group_members += 1 uid = user.get('uid') if user and uid: if uid not in user_by_uid: user_by_uid[uid] = user total_group_users += 1 user_groups = user_by_uid[uid]['groups'] if group not in user_groups: user_groups.append(group) self.logger.debug('Group %s members: %d users: %d', group, total_group_members, total_group_users) return six.itervalues(user_by_uid)
def add_user(self, attributes): ''' :type attributes: dict ''' if self.identity_type == user_sync.identity_type.ADOBEID_IDENTITY_TYPE: email = self.email if self.email else self.username if not email: errorMessage = "ERROR: you must specify an email with an Adobe ID" raise AssertionException(errorMessage) params = self.convert_user_attributes_to_params({'email': email}) else: params = self.convert_user_attributes_to_params(attributes) onConflictValue = None option = params.pop('option', None) if (option == "updateIfAlreadyExists"): onConflictValue = umapi_client.IfAlreadyExistsOptions.updateIfAlreadyExists elif (option == "ignoreIfAlreadyExists"): onConflictValue = umapi_client.IfAlreadyExistsOptions.ignoreIfAlreadyExists if (onConflictValue != None): params['on_conflict'] = onConflictValue self.do_list.append(('create', params))
def _admin_role_mapping(sync_config): admin_roles = sync_config.get_list('admin_roles', True) if admin_roles is None: return {} mapped_admin_roles = {} for mapping in admin_roles: sign_role = mapping.get('sign_role') if sign_role is None: raise AssertionException( "must define a Sign role in admin role mapping") adobe_groups = mapping.get('adobe_groups') if adobe_groups is None or not len(adobe_groups): continue for g in adobe_groups: group = AdobeGroup.create(g) group_name = group.group_name.lower() if group.umapi_name not in mapped_admin_roles: mapped_admin_roles[group.umapi_name] = {} if group_name not in mapped_admin_roles[group.umapi_name]: mapped_admin_roles[group.umapi_name][group_name] = set() mapped_admin_roles[group.umapi_name][group_name].add(sign_role) return mapped_admin_roles
def open_csv_file(name, mode, encoding=None): """ :type name: str :type mode: str :type encoding: str, but ignored in py2 :rtype file """ try: if mode == 'r': if is_py2(): return open(str(name), 'rb', buffering=1) else: kwargs = dict(buffering=1, newline='', encoding=encoding) return open(str(name), 'r', **kwargs) elif mode == 'w': if is_py2(): return open(str(name), 'wb') else: kwargs = dict(newline='') return open(str(name), 'w', **kwargs) else: raise ValueError("File mode (%s) must be 'r' or 'w'" % mode) except IOError as e: raise AssertionException("Can't open file '%s': %s" % (name, e))
def begin_work(config_loader): ''' :type config_loader: user_sync.config.ConfigLoader ''' directory_groups = config_loader.get_directory_groups() primary_umapi_config, secondary_umapi_configs = config_loader.get_umapi_options( ) rule_config = config_loader.get_rule_options() # process mapped configuration after the directory groups have been loaded, as mapped setting depends on this. if (rule_config['directory_group_mapped']): rule_config['directory_group_filter'] = set( directory_groups.iterkeys()) # make sure that all the adobe groups are from known umapi connector names referenced_umapi_names = set() for groups in directory_groups.itervalues(): for group in groups: umapi_name = group.umapi_name if (umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME): referenced_umapi_names.add(umapi_name) referenced_umapi_names.difference_update( secondary_umapi_configs.iterkeys()) if (len(referenced_umapi_names) > 0): raise AssertionException( 'Adobe groups reference unknown umapi connectors: %s' % referenced_umapi_names) directory_connector = None directory_connector_options = None directory_connector_module_name = config_loader.get_directory_connector_module_name( ) if (directory_connector_module_name != None): directory_connector_module = __import__( directory_connector_module_name, fromlist=['']) directory_connector = user_sync.connector.directory.DirectoryConnector( directory_connector_module) directory_connector_options = config_loader.get_directory_connector_options( directory_connector.name) config_loader.check_unused_config_keys() if (directory_connector != None and directory_connector_options != None): # specify the default user_identity_type if it's not already specified in the options if 'user_identity_type' not in directory_connector_options: directory_connector_options['user_identity_type'] = rule_config[ 'new_account_type'] directory_connector.initialize(directory_connector_options) primary_name = '.primary' if secondary_umapi_configs else '' umapi_primary_connector = user_sync.connector.umapi.UmapiConnector( primary_name, primary_umapi_config) umapi_other_connectors = {} for secondary_umapi_name, secondary_config in secondary_umapi_configs.iteritems( ): umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector( ".secondary.%s" % secondary_umapi_name, secondary_config) umapi_other_connectors[secondary_umapi_name] = umapi_secondary_conector umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector, umapi_other_connectors) rule_processor = user_sync.rules.RuleProcessor(rule_config) if (len(directory_groups) == 0 and rule_processor.will_manage_groups()): logger.warn('no groups mapped in config file') rule_processor.run(directory_groups, directory_connector, umapi_connectors)
def get_rule_options(self): ''' Return a dict representing options for RuleProcessor. ''' # process directory configuration options new_account_type = None default_country_code = None directory_config = self.main_config.get_dict_config( 'directory_users', True) if directory_config: new_account_type = directory_config.get_string( 'user_identity_type', True) new_account_type = user_sync.identity_type.parse_identity_type( new_account_type) default_country_code = directory_config.get_string( 'default_country_code', True) if not new_account_type: new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE self.logger.debug("Using default for new_account_type: %s", new_account_type) # process exclusion configuration options exclude_identity_types = exclude_identity_type_names = [] exclude_users = exclude_users_regexps = [] exclude_groups = exclude_group_names = [] adobe_config = self.main_config.get_dict_config('adobe_users', True) if adobe_config: exclude_identity_type_names = adobe_config.get_list( 'exclude_identity_types', True) or [] exclude_users_regexps = adobe_config.get_list( 'exclude_users', True) or [] exclude_group_names = adobe_config.get_list( 'exclude_adobe_groups', True) or [] for name in exclude_identity_type_names: message_format = 'Illegal value in exclude_identity_types: %s' identity_type = user_sync.identity_type.parse_identity_type( name, message_format) exclude_identity_types.append(identity_type) for regexp in exclude_users_regexps: try: # add "match begin" and "match end" markers to ensure complete match # and compile the patterns because we will use them over and over exclude_users.append( re.compile(r'\A' + regexp + r'\Z', re.UNICODE)) except re.error as e: validation_message = ( 'Illegal regular expression (%s) in %s: %s' % (regexp, 'exclude_identity_types', e)) raise AssertionException(validation_message) for name in exclude_group_names: group = user_sync.rules.AdobeGroup.create(name) if not group or group.get_umapi_name( ) != user_sync.rules.PRIMARY_UMAPI_NAME: validation_message = 'Illegal value for %s in config file: %s' % ( 'exclude_groups', name) if not group: validation_message += ' (Not a legal group name)' else: validation_message += ' (Can only exclude groups in primary organization)' raise AssertionException(validation_message) exclude_groups.append(group.get_group_name()) # get the limits limits_config = self.main_config.get_dict_config('limits') max_adobe_only_users = limits_config.get_int('max_adobe_only_users') # now get the directory extension, if any after_mapping_hook = None extended_attributes = None extension_config = self.get_directory_extension_options() if extension_config: after_mapping_hook_text = extension_config.get_string( 'after_mapping_hook') after_mapping_hook = compile(after_mapping_hook_text, '<per-user after-mapping-hook>', 'exec') extended_attributes = extension_config.get_list( 'extended_attributes') # declaration of extended adobe groups: this is needed for two reasons: # 1. it allows validation of group names, and matching them to adobe groups # 2. it allows removal of adobe groups not assigned by the hook for extended_adobe_group in extension_config.get_list( 'extended_adobe_groups'): group = user_sync.rules.AdobeGroup.create(extended_adobe_group) if group is None: message = 'Extension contains illegal extended_adobe_group spec: ' + str( extended_adobe_group) raise AssertionException(message) options = self.options result = { # these are in alphabetical order! Always add new ones that way! 'after_mapping_hook': after_mapping_hook, 'default_country_code': default_country_code, 'delete_strays': options['delete_strays'], 'directory_group_filter': options['directory_group_filter'], 'directory_group_mapped': options['directory_group_mapped'], 'disentitle_strays': options['disentitle_strays'], 'exclude_groups': exclude_groups, 'exclude_identity_types': exclude_identity_types, 'exclude_strays': options['exclude_strays'], 'exclude_users': exclude_users, 'extended_attributes': extended_attributes, 'manage_groups': options['manage_groups'], 'max_adobe_only_users': max_adobe_only_users, 'new_account_type': new_account_type, 'remove_strays': options['remove_strays'], 'stray_list_input_path': options['stray_list_input_path'], 'stray_list_output_path': options['stray_list_output_path'], 'test_mode': options['test_mode'], 'update_user_info': options['update_user_info'], 'username_filter_regex': options['username_filter_regex'], } return result
def __init__(self, caller_options): caller_config = user_sync.config.DictConfig( '%s configuration' % self.name, caller_options) options = self.get_options(caller_config) self.options = options self.logger = logger = user_sync.connector.helper.create_logger( options) logger.debug('%s initialized with options: %s', self.name, options) LDAPValueFormatter.encoding = options['string_encoding'] self.user_identity_type = user_sync.identity_type.parse_identity_type( options['user_identity_type']) self.user_identity_type_formatter = LDAPValueFormatter( options['user_identity_type_format']) self.user_email_formatter = LDAPValueFormatter( options['user_email_format']) self.user_username_formatter = LDAPValueFormatter( options['user_username_format']) self.user_domain_formatter = LDAPValueFormatter( options['user_domain_format']) self.user_given_name_formatter = LDAPValueFormatter( options['user_given_name_format']) self.user_surname_formatter = LDAPValueFormatter( options['user_surname_format']) self.user_country_code_formatter = LDAPValueFormatter( options['user_country_code_format']) auth_method = options['authentication_method'].lower() auth_cred_required = ['simple', 'ntlm'] if options['username'] is not None: if auth_method in auth_cred_required: password = caller_config.get_credential( 'password', options['username']) else: # Ignore specified credential if authentication method is either 'Kerberos' or 'Anonymous' raise AssertionException( "'username' and 'password' are not allowed when 'authentication_method' is '%s" % auth_method) else: # override authentication method to anonymous if username is not specified if auth_method != 'anonymous' and auth_method != 'kerberos': auth_method = 'anonymous' logger.info( "Username not specified, overriding authentication method to 'anonymous'" ) # this check must come after we get the password value caller_config.report_unused_values(logger) if auth_method != 'kerberos': from ldap3 import Connection if auth_method == 'anonymous': auth = {'authentication': ldap3.ANONYMOUS} logger.debug( 'Connecting to: %s - Authentication Method: ANONYMOUS', options['host']) elif auth_method == 'simple': auth = { 'authentication': ldap3.SIMPLE, 'user': six.text_type(options['username']), 'password': six.text_type(password) } logger.debug( 'Connecting to: %s - Authentication Method: SIMPLE using username: %s', options['host'], options['username']) elif auth_method == 'ntlm': auth = { 'authentication': ldap3.NTLM, 'user': six.text_type(options['username']), 'password': six.text_type(password) } logger.debug( 'Connecting to: %s - Authentication Method: NTLM using username: %s', options['host'], options['username']) elif auth_method == 'kerberos': if (platform.system() == 'Windows'): from .ldap3_extended.Connection import Connection auth = { 'authentication': ldap3.SASL, 'sasl_mechanism': ldap3.GSSAPI } logger.debug( 'Connecting to: %s - Authentication Method: Kerberos', options['host']) else: raise AssertionException( 'Kerberos Authentication Method is not supported on this OS. Windows Only' ) else: raise AssertionException( 'LDAP Authentication Method is not supported: %s' % auth_method) tls = None auto_bind = ldap3.AUTO_BIND_NO_TLS if options['require_tls_cert']: tls = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLSv1_2) auto_bind = ldap3.AUTO_BIND_TLS_BEFORE_BIND try: server = ldap3.Server(host=options['host'], allowed_referral_hosts=True, tls=tls) connection = Connection(server, auto_bind=auto_bind, read_only=True, **auth) except Exception as e: raise AssertionException('LDAP connection failure: %s' % e) self.connection = connection logger.debug('Connected as %s', connection.extend.standard.who_am_i()) self.user_by_dn = {} self.additional_group_filters = None
def load_users_and_groups(self, groups, extended_attributes, all_users): """ :type groups: list(str) :type extended_attributes: list(str) :type all_users: bool :rtype (bool, iterable(dict)) """ options = self.options user = {} base_dn = six.text_type(options['base_dn']) all_users_filter = six.text_type(options['all_users_filter']) group_member_filter_format = six.text_type( options['group_member_filter_format']) grouped_user_records = {} if options['two_steps_enabled']: group_member_attribute_name = six.text_type( options['two_steps_lookup']['group_member_attribute_name']) # save all the users to memory for faster 2-steps lookup or all_users process if all_users: try: all_users_records = dict( self.iter_users(base_dn, all_users_filter, extended_attributes)) except Exception as e: raise AssertionException( 'Unexpected LDAP failure reading all users: %s' % e) # for each group that's required, do one search for the users of that group for group in groups: group_users = 0 group_dn = self.find_ldap_group_dn(group) if not group_dn: self.logger.warning("No group found for: %s", group) continue group_member_subfilter = self.format_ldap_query_string( group_member_filter_format, group_dn=group_dn) if not group_member_subfilter.startswith('('): group_member_subfilter = six.text_type( '(') + group_member_subfilter + six.text_type(')') user_subfilter = all_users_filter if not user_subfilter.startswith('('): user_subfilter = six.text_type( '(') + user_subfilter + six.text_type(')') group_user_filter = six.text_type( '(&' ) + group_member_subfilter + user_subfilter + six.text_type(')') group_users = 0 try: if options['two_steps_enabled']: for user_dn in self.iter_group_member_dns( group_dn, group_member_attribute_name): # check to make sure user_dn are within the base_dn scope if self.is_dn_within_base_dn_scope(base_dn, user_dn): # replace base_dn with user_dn and filter with all_users_filter to do user lookup based on DN result = list( self.iter_users(user_dn, all_users_filter, extended_attributes)) if result: # iter_users should only return 1 user when doing two_steps lookup. if len(result) > 1: raise AssertionException( "Unexpected multiple LDAP object found in 'two_steps_lookup' mode for: %s" % user_dn) else: user = result[0][1] user['groups'].append(group) group_users += 1 grouped_user_records[user_dn] = user else: for user_dn, user in self.iter_users( base_dn, group_user_filter, extended_attributes): user['groups'].append(group) group_users += 1 grouped_user_records[user_dn] = user except Exception as e: raise AssertionException( 'Unexpected LDAP failure reading group members: %s' % e) self.logger.debug('Count of users in group "%s": %d', group, group_users) # if all users are requested, do an additional search for all of them if all_users: ungrouped_users = 0 grouped_users = 0 try: for user_dn, user in self.iter_users(base_dn, all_users_filter, extended_attributes): if not user['groups']: ungrouped_users += 1 else: grouped_users += 1 if groups: self.logger.debug('Count of users in any groups: %d', grouped_users) self.logger.debug('Count of users not in any groups: %d', ungrouped_users) except Exception as e: raise AssertionException( 'Unexpected LDAP failure reading all users: %s' % e) self.logger.debug('Total users loaded: %d', len(self.user_by_dn)) return six.itervalues(self.user_by_dn)
def __init__(self, name, caller_options): ''' :type name: str :type caller_options: dict ''' caller_config = user_sync.config.DictConfig( '"%s umapi options"' % name, caller_options) builder = user_sync.config.OptionsBuilder(caller_config) builder.set_string_value('logger_name', 'umapi' + name) builder.set_bool_value('test_mode', False) options = builder.get_options() server_config = caller_config.get_dict_config('server', True) server_builder = user_sync.config.OptionsBuilder(server_config) server_builder.set_string_value('host', 'usermanagement.adobe.io') server_builder.set_string_value('endpoint', '/v2/usermanagement') server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com') server_builder.set_string_value('ims_endpoint_jwt', '/ims/exchange/jwt') options['server'] = server_options = server_builder.get_options() enterprise_config = caller_config.get_dict_config('enterprise') enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config) enterprise_builder.require_string_value('org_id') enterprise_builder.require_string_value('api_key') enterprise_builder.require_string_value('client_secret') enterprise_builder.require_string_value('tech_acct') enterprise_builder.require_string_value('priv_key_path') options[ 'enterprise'] = enterprise_options = enterprise_builder.get_options( ) self.options = options self.logger = logger = helper.create_logger(options) caller_config.report_unused_values(logger) ims_host = server_options['ims_host'] self.org_id = org_id = enterprise_options['org_id'] api_key = enterprise_options['api_key'] private_key_file_path = enterprise_options['priv_key_path'] um_endpoint = "https://" + server_options['host'] + server_options[ 'endpoint'] logger.debug( 'Creating connection for org id: "%s" using private key file: "%s"', org_id, private_key_file_path) auth_dict = { "org_id": org_id, "tech_acct_id": enterprise_options['tech_acct'], "api_key": api_key, "client_secret": enterprise_options['client_secret'], "private_key_file": private_key_file_path } try: self.connection = connection = umapi_client.Connection( org_id=org_id, auth_dict=auth_dict, ims_host=ims_host, ims_endpoint_jwt=server_options['ims_endpoint_jwt'], user_management_endpoint=um_endpoint, test_mode=options['test_mode'], user_agent="user-sync/" + APP_VERSION, logger=self.logger, ) except Exception as e: raise AssertionException( "UMAPI connection to org id '%s' failed: %s" % (org_id, e)) logger.debug('API initialized on: %s', um_endpoint) self.action_manager = ActionManager(connection, org_id, logger)
def get_rule_options(self): """ Return a dict representing options for RuleProcessor. """ options = user_sync.rules.RuleProcessor.default_options options.update(self.invocation_options) # process directory configuration options directory_config = self.main_config.get_dict_config( 'directory_users', True) if directory_config: # account type new_account_type = directory_config.get_string( 'user_identity_type', True) new_account_type = user_sync.identity_type.parse_identity_type( new_account_type) if new_account_type: options['new_account_type'] = new_account_type else: self.logger.debug("Using default for new_account_type: %s", options['new_account_type']) # country code default_country_code = directory_config.get_string( 'default_country_code', True) if default_country_code: options['default_country_code'] = default_country_code # process exclusion configuration options adobe_config = self.main_config.get_dict_config('adobe_users', True) if adobe_config: exclude_identity_type_names = adobe_config.get_list( 'exclude_identity_types', True) if exclude_identity_type_names: exclude_identity_types = [] for name in exclude_identity_type_names: message_format = 'Illegal value in exclude_identity_types: %s' identity_type = user_sync.identity_type.parse_identity_type( name, message_format) exclude_identity_types.append(identity_type) options['exclude_identity_types'] = exclude_identity_types exclude_users_regexps = adobe_config.get_list( 'exclude_users', True) if exclude_users_regexps: exclude_users = [] for regexp in exclude_users_regexps: try: # add "match begin" and "match end" markers to ensure complete match # and compile the patterns because we will use them over and over exclude_users.append( re.compile(r'\A' + regexp + r'\Z', re.UNICODE)) except re.error as e: validation_message = ( 'Illegal regular expression (%s) in %s: %s' % (regexp, 'exclude_identity_types', e)) raise AssertionException(validation_message) options['exclude_users'] = exclude_users exclude_group_names = adobe_config.get_list( 'exclude_adobe_groups', True) or [] if exclude_group_names: exclude_groups = [] for name in exclude_group_names: group = user_sync.rules.AdobeGroup.create(name) if not group or group.get_umapi_name( ) != user_sync.rules.PRIMARY_UMAPI_NAME: validation_message = 'Illegal value for %s in config file: %s' % ( 'exclude_groups', name) if not group: validation_message += ' (Not a legal group name)' else: validation_message += ' (Can only exclude groups in primary organization)' raise AssertionException(validation_message) exclude_groups.append(group.get_group_name()) options['exclude_groups'] = exclude_groups # get the limits limits_config = self.main_config.get_dict_config('limits') options['max_adobe_only_users'] = limits_config.get_int( 'max_adobe_only_users') # now get the directory extension, if any extension_config = self.get_directory_extension_options() if extension_config: after_mapping_hook_text = extension_config.get_string( 'after_mapping_hook') options['after_mapping_hook'] = compile( after_mapping_hook_text, '<per-user after-mapping-hook>', 'exec') options['extended_attributes'] = extension_config.get_list( 'extended_attributes') # declaration of extended adobe groups: this is needed for two reasons: # 1. it allows validation of group names, and matching them to adobe groups # 2. it allows removal of adobe groups not assigned by the hook for extended_adobe_group in extension_config.get_list( 'extended_adobe_groups'): group = user_sync.rules.AdobeGroup.create(extended_adobe_group) if group is None: message = 'Extension contains illegal extended_adobe_group spec: ' + str( extended_adobe_group) raise AssertionException(message) # set the directory group filter from the mapping, if requested. # This must come late, after any prior adds to the mapping from other parameters. if options.get('directory_group_mapped'): options['directory_group_filter'] = set( six.iterkeys(self.directory_groups)) return options
def begin_work(config_loader): """ :type config_loader: user_sync.config.ConfigLoader """ directory_groups = config_loader.get_directory_groups() rule_config = config_loader.get_rule_options() # make sure that all the adobe groups are from known umapi connector names primary_umapi_config, secondary_umapi_configs = config_loader.get_umapi_options( ) referenced_umapi_names = set() for groups in six.itervalues(directory_groups): for group in groups: umapi_name = group.umapi_name if umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME: referenced_umapi_names.add(umapi_name) referenced_umapi_names.difference_update( six.iterkeys(secondary_umapi_configs)) if len(referenced_umapi_names) > 0: raise AssertionException( 'Adobe groups reference unknown umapi connectors: %s' % referenced_umapi_names) directory_connector = None directory_connector_options = None directory_connector_module_name = config_loader.get_directory_connector_module_name( ) if directory_connector_module_name is not None: directory_connector_module = __import__( directory_connector_module_name, fromlist=['']) directory_connector = user_sync.connector.directory.DirectoryConnector( directory_connector_module) directory_connector_options = config_loader.get_directory_connector_options( directory_connector.name) config_loader.check_unused_config_keys() if directory_connector is not None and directory_connector_options is not None: # specify the default user_identity_type if it's not already specified in the options if 'user_identity_type' not in directory_connector_options: directory_connector_options['user_identity_type'] = rule_config[ 'new_account_type'] directory_connector.initialize(directory_connector_options) primary_name = '.primary' if secondary_umapi_configs else '' umapi_primary_connector = user_sync.connector.umapi.UmapiConnector( primary_name, primary_umapi_config) umapi_other_connectors = {} for secondary_umapi_name, secondary_config in six.iteritems( secondary_umapi_configs): umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector( ".secondary.%s" % secondary_umapi_name, secondary_config) umapi_other_connectors[secondary_umapi_name] = umapi_secondary_conector umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector, umapi_other_connectors) rule_processor = user_sync.rules.RuleProcessor(rule_config) if len(directory_groups) == 0 and rule_processor.will_process_groups(): logger.warning( 'No group mapping specified in configuration but --process-groups requested on command line' ) rule_processor.run(directory_groups, directory_connector, umapi_connectors)
def iter_user_groups(self): try: for g in umapi_client.UserGroupsQuery(self.connection): yield g except umapi_client.UnavailableError as e: raise AssertionException("Error contacting UMAPI server: %s" % e)
def create_assertion_error(self, message): return AssertionException("%s in: %s" % (message, self.get_full_scope()))
def __init__(self, caller_options): caller_config = user_sync.config.DictConfig( '%s configuration' % self.name, caller_options) builder = user_sync.config.OptionsBuilder(caller_config) builder.set_string_value('group_filter_format', '{group}') builder.set_string_value('all_users_filter', 'user.status == "ACTIVE"') builder.set_string_value('string_encoding', 'utf8') builder.set_string_value('user_identity_type_format', None) builder.set_string_value('user_email_format', six.text_type('{email}')) builder.set_string_value('user_username_format', None) builder.set_string_value('user_domain_format', None) builder.set_string_value('user_given_name_format', six.text_type('{firstName}')) builder.set_string_value('user_surname_format', six.text_type('{lastName}')) builder.set_string_value('user_country_code_format', six.text_type('{countryCode}')) builder.set_string_value('user_identity_type', None) builder.set_string_value('logger_name', self.name) host = builder.require_string_value('host') api_token = builder.require_string_value('api_token') options = builder.get_options() OKTAValueFormatter.encoding = options['string_encoding'] self.user_identity_type = user_sync.identity_type.parse_identity_type( options['user_identity_type']) self.user_identity_type_formatter = OKTAValueFormatter( options['user_identity_type_format']) self.user_email_formatter = OKTAValueFormatter( options['user_email_format']) self.user_username_formatter = OKTAValueFormatter( options['user_username_format']) self.user_domain_formatter = OKTAValueFormatter( options['user_domain_format']) self.user_given_name_formatter = OKTAValueFormatter( options['user_given_name_format']) self.user_surname_formatter = OKTAValueFormatter( options['user_surname_format']) self.user_country_code_formatter = OKTAValueFormatter( options['user_country_code_format']) self.users_client = None self.groups_client = None self.logger = logger = user_sync.connector.helper.create_logger( options) self.user_identity_type = user_sync.identity_type.parse_identity_type( options['user_identity_type']) self.options = options caller_config.report_unused_values(logger) if not host.startswith('https://'): if "://" in host: raise AssertionException("Okta protocol must be https") host = "https://" + host self.user_by_uid = {} logger.debug('%s initialized with options: %s', self.name, options) logger.info('Connecting to: %s', host) try: self.users_client = okta.UsersClient(host, api_token) self.groups_client = okta.UserGroupsClient(host, api_token) except OktaError as e: raise AssertionException("Error connecting to Okta: %s" % e) logger.info('Connected')
def load_invocation_options(self): """Merge the invocation option defaults with overrides from the main config and the command line. :rtype: dict """ options = self.invocation_defaults # get overrides from the main config invocation_config = self.main_config.get_dict_config( 'invocation_defaults', True) if invocation_config: for k, v in six.iteritems(self.invocation_defaults): if isinstance(v, bool): val = invocation_config.get_bool(k, True) if val is not None: options[k] = val elif isinstance(v, list): val = invocation_config.get_list(k, True) if val: options[k] = val else: val = invocation_config.get_string(k, True) if val: options[k] = val # now process command line options. the order of these is important, # because options processed later depend on the values of those processed earlier # --connector connector_spec = self.args['connector'] or options['connector'] connector_type = user_sync.helper.normalize_string(connector_spec[0]) if connector_type in ["ldap", "okta"]: if len(connector_spec) > 1: raise AssertionException( 'Must not specify a file (%s) with connector type %s' % (connector_spec[0], connector_type)) options['directory_connector_type'] = connector_type elif connector_type == "csv": if len(connector_spec) != 2: raise AssertionException( "You must specify a single file with connector type csv") options['directory_connector_type'] = 'csv' options['directory_connector_overridden_options'] = { 'file_path': connector_spec[1] } else: raise AssertionException('Unknown connector type: %s' % connector_type) # --process-groups if self.args['process_groups'] is not None: options['process_groups'] = self.args['process_groups'] # --strategy if user_sync.helper.normalize_string(self.args['strategy'] or options['strategy']) == 'push': options['strategy'] = 'push' # --test-mode if self.args['test_mode'] is not None: options['test_mode'] = self.args['test_mode'] # --update-user-info if self.args['update_user_info'] is not None: options['update_user_info'] = self.args['update_user_info'] # --adobe-only-user-action if options['strategy'] == 'push': if self.args['adobe_only_user_action']: raise AssertionException( 'You cannot specify --adobe-only-user-action when using "push" strategy' ) self.logger.info( "Strategy push: ignoring default adobe-only-user-action") else: adobe_action_spec = self.args['adobe_only_user_action'] or options[ 'adobe_only_user_action'] adobe_action = user_sync.helper.normalize_string( adobe_action_spec[0]) if adobe_action == 'preserve': pass # no option settings needed elif adobe_action == 'exclude': options['exclude_strays'] = True elif adobe_action == 'write-file': if len(adobe_action_spec) != 2: raise AssertionException( 'You must specify a single file for adobe-only-user-action "write-file"' ) options['stray_list_output_path'] = adobe_action_spec[1] elif adobe_action == 'delete': options['delete_strays'] = True elif adobe_action == 'remove': options['remove_strays'] = True elif adobe_action == 'remove-adobe-groups': options['disentitle_strays'] = True else: raise AssertionException( 'Unknown option "%s" for adobe-only-user-action' % adobe_action) # --users and --adobe-only-user-list conflict with each other, so we need to disambiguate. # Argument specifications override configuration options, so you must have one or the other # either as an argument or as a configured default. if self.args['users'] and self.args['adobe_only_user_list']: # specifying both --users and --adobe-only-user-list is an error raise AssertionException( 'You cannot specify both a --users arg and an --adobe-only-user-list arg' ) elif self.args['users']: # specifying --users overrides the configuration file default for this option options['users'] = self.args['users'] users_spec = self.args['users'] stray_list_input_path = None elif self.args['adobe_only_user_list']: # specifying --adobe-only-user-list overrides the configuration file default for --users if options['strategy'] == 'push': raise AssertionException( 'You cannot specify --adobe-only-user-list when using "push" strategy' ) users_spec = None stray_list_input_path = self.args['adobe_only_user_list'] elif options['users'] and options['adobe_only_user_list']: raise AssertionException( 'You cannot configure both a default "users" option (%s) ' 'and a default "adobe-only-user-list" option (%s)' % (' '.join(options['users']), options['adobe_only_user_list'])) elif options['users']: users_spec = options['users'] stray_list_input_path = None elif options['adobe_only_user_list']: users_spec = None stray_list_input_path = options['adobe_only_user_list'] else: raise AssertionException( 'You must specify either a "users" option or an "adobe-only-user-list" option.' ) # --users if users_spec: users_action = user_sync.helper.normalize_string(users_spec[0]) if users_action == 'all': if options['directory_connector_type'] == 'okta': raise AssertionException( 'Okta connector module does not support "--users all"') elif users_action == 'file': if options['directory_connector_type'] == 'csv': raise AssertionException( 'You cannot specify file input with both "users" and "connector" options' ) if len(users_spec) != 2: raise AssertionException( 'You must specify the file to read when using the users "file" option' ) options['directory_connector_type'] = 'csv' options['directory_connector_overridden_options'] = { 'file_path': users_spec[1] } elif users_action == 'mapped': options['directory_group_mapped'] = True elif users_action == 'group': if len(users_spec) != 2: raise AssertionException( 'You must specify the groups to read when using the users "group" option' ) options['directory_group_filter'] = users_spec[1].split(',') else: raise AssertionException('Unknown option "%s" for users' % users_action) # --adobe-only-user-list if options['strategy'] == 'push': self.logger.info( "Strategy push: ignoring default adobe-only-user-list") elif stray_list_input_path: if options.get('stray_list_output_path'): raise AssertionException( 'You cannot specify both an adobe-only-user-list (%s) and ' 'an adobe-only-user-action of "write-file"') # don't read the directory when processing from the stray list self.logger.info( 'adobe-only-user-list specified, so not reading or comparing directory and Adobe users' ) options['stray_list_input_path'] = stray_list_input_path # --user-filter if stray_list_input_path: if self.args['user_filter']: raise AssertionException( 'You cannot specify --user-filter when using an adobe-only-user-list' ) self.logger.info( "adobe-only-user-list specified, so ignoring default user filter specification" ) else: username_filter_pattern = self.args['user_filter'] or options[ 'user_filter'] if username_filter_pattern: try: compiled_expression = re.compile( r'\A' + username_filter_pattern + r'\Z', re.IGNORECASE) except Exception as e: raise AssertionException( "Bad regular expression in user filter: %s reason: %s" % (username_filter_pattern, e)) options['username_filter_regex'] = compiled_expression return options
def load_from_yaml(cls, filename, path_keys): ''' loads a yaml file, processes the resulting dict to adapt values for keys (the path to which is defined in path_keys) to a value that represents a file reference relative to the source file being loaded, and returns the processed dict. :param filename: the file to load yaml from :param path_keys: a dict whose keys are "path_keys" such as /key1/key2/key3 and whose values are tuples: (must_exist, can_have_subdict, default_val) which are options on the value of the key whose values are path expanded: must the path exist, can it be a list of paths that contains sub-dictionaries whose values are paths, and does the key have a default value so that must be added to the dictionary if there is not already a value found. ''' if filename.startswith('$(') and filename.endswith(')'): # it's a command line to execute and read standard output dir_end = filename.index(']') if filename.startswith('$([') and dir_end > 0: dir = filename[3:dir_end] cmd = filename[dir_end + 1:-1] else: dir = os.path.abspath(".") cmd = filename[3:-1] try: bytes = subprocess.check_output(cmd, cwd=dir, shell=True) yml = yaml.load(bytes.decode(cls.config_encoding, 'strict')) except subprocess.CalledProcessError as e: raise AssertionException( "Error executing process '%s' in dir '%s': %s" % (cmd, dir, e)) except UnicodeDecodeError as e: raise AssertionException( 'Encoding error in process output: %s' % e) except yaml.error.MarkedYAMLError as e: raise AssertionException( 'Error parsing process YAML data: %s' % e) else: # it's a pathname to a configuration file to read cls.filepath = os.path.abspath(filename) if not os.path.isfile(cls.filepath): raise AssertionException('No such configuration file: %s' % (cls.filepath, )) cls.filename = os.path.split(cls.filepath)[1] cls.dirpath = os.path.dirname(cls.filepath) try: with open(filename, 'rb', 1) as input_file: bytes = input_file.read() yml = yaml.load(bytes.decode(cls.config_encoding, 'strict')) except IOError as e: # if a file operation error occurred while loading the # configuration file, swallow up the exception and re-raise it # as an configuration loader exception. raise AssertionException( "Error reading configuration file '%s': %s" % (cls.filepath, e)) except UnicodeDecodeError as e: # as above, but in case of encoding errors raise AssertionException( "Encoding error in configuration file '%s: %s" % (cls.filepath, e)) except yaml.error.MarkedYAMLError as e: # as above, but in case of parse errors raise AssertionException( "Error parsing configuration file '%s': %s" % (cls.filepath, e)) # process the content of the dict for path_key, options in path_keys.iteritems(): cls.key_path = path_key keys = path_key.split('/') cls.process_path_key(yml, keys, 1, *options) return yml
def create_config_loader_options(args): ''' This is where all the command-line arguments get set as options in the main config so that it appears as if they were loaded as part of the main configuration file. If you add an option that is supposed to be set from the command line here, you had better make sure you add it to the ones read in get_rule_options. :param args: the command-line args as parsed :return: the configured options for the config loader. ''' config_options = { 'delete_strays': False, 'directory_connector_module_name': None, 'directory_connector_overridden_options': None, 'directory_group_filter': None, 'directory_group_mapped': False, 'disentitle_strays': False, 'exclude_strays': False, 'manage_groups': args.manage_groups, 'remove_strays': False, 'stray_list_input_path': None, 'stray_list_output_path': None, 'test_mode': args.test_mode, 'update_user_info': args.update_user_info, 'username_filter_regex': None, } # --users users_args = args.users if users_args is not None: users_action = None if len( users_args) == 0 else user_sync.helper.normalize_string( users_args.pop(0)) if (users_action == None or users_action == 'all'): config_options[ 'directory_connector_module_name'] = 'user_sync.connector.directory_ldap' elif (users_action == 'file'): if len(users_args) == 0: raise AssertionException( 'Missing file path for --users %s [file_path]' % users_action) config_options[ 'directory_connector_module_name'] = 'user_sync.connector.directory_csv' config_options['directory_connector_overridden_options'] = { 'file_path': users_args.pop(0) } elif (users_action == 'mapped'): config_options[ 'directory_connector_module_name'] = 'user_sync.connector.directory_ldap' config_options['directory_group_mapped'] = True elif (users_action == 'group'): if len(users_args) == 0: raise AssertionException( 'Missing groups for --users %s [groups]' % users_action) config_options[ 'directory_connector_module_name'] = 'user_sync.connector.directory_ldap' config_options['directory_group_filter'] = users_args.pop(0).split( ',') else: raise AssertionException('Unknown argument --users %s' % users_action) username_filter_pattern = args.username_filter_pattern if (username_filter_pattern): try: compiled_expression = re.compile( r'\A' + username_filter_pattern + r'\Z', re.IGNORECASE) except Exception as e: raise AssertionException( "Bad regular expression for --user-filter: %s reason: %s" % (username_filter_pattern, e.message)) config_options['username_filter_regex'] = compiled_expression # --adobe-only-user-action adobe_action_args = args.adobe_only_user_action if adobe_action_args is not None: adobe_action = None if not adobe_action_args else user_sync.helper.normalize_string( adobe_action_args.pop(0)) if (adobe_action == None or adobe_action == 'preserve'): pass # no option settings needed elif (adobe_action == 'exclude'): config_options['exclude_strays'] = True elif (adobe_action == 'write-file'): if not adobe_action_args: raise AssertionException( 'Missing file path for --adobe-only-user-action %s [file_path]' % adobe_action) config_options['stray_list_output_path'] = adobe_action_args.pop(0) elif (adobe_action == 'delete'): config_options['delete_strays'] = True elif (adobe_action == 'remove'): config_options['remove_strays'] = True elif (adobe_action == 'remove-adobe-groups'): config_options['disentitle_strays'] = True else: raise AssertionException( 'Unknown argument --adobe-only-user-action %s' % adobe_action) # --adobe-only-user-list stray_list_input_path = args.stray_list_input_path if stray_list_input_path: if users_args is not None: raise AssertionException( 'You cannot specify both --users and --adobe-only-user-list') if config_options.get('stray_list_output_path'): raise AssertionException( 'You cannot specify both --adobe-only-user-list and --output-adobe-users' ) # don't read the directory when processing from the stray list config_options['directory_connector_module_name'] = None logger.info( '--adobe-only-user-list specified, so not reading or comparing directory and Adobe users' ) config_options['stray_list_input_path'] = stray_list_input_path return config_options
def __init__(self, name, caller_options): ''' :type name: str :type caller_options: dict ''' self.name = 'umapi' + name caller_config = user_sync.config.DictConfig( self.name + ' configuration', caller_options) builder = user_sync.config.OptionsBuilder(caller_config) builder.set_string_value('logger_name', self.name) builder.set_bool_value('test_mode', False) options = builder.get_options() server_config = caller_config.get_dict_config('server', True) server_builder = user_sync.config.OptionsBuilder(server_config) server_builder.set_string_value('host', 'usermanagement.adobe.io') server_builder.set_string_value('endpoint', '/v2/usermanagement') server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com') server_builder.set_string_value('ims_endpoint_jwt', '/ims/exchange/jwt') options['server'] = server_options = server_builder.get_options() enterprise_config = caller_config.get_dict_config('enterprise') enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config) enterprise_builder.require_string_value('org_id') enterprise_builder.require_string_value('tech_acct') options[ 'enterprise'] = enterprise_options = enterprise_builder.get_options( ) self.options = options self.logger = logger = helper.create_logger(options) if server_config: server_config.report_unused_values(logger) logger.debug('UMAPI initialized with options: %s', options) # set up the auth dict for umapi-client ims_host = server_options['ims_host'] self.org_id = org_id = enterprise_options['org_id'] auth_dict = { 'org_id': org_id, 'tech_acct_id': enterprise_options['tech_acct'], 'api_key': enterprise_config.get_credential('api_key', org_id), 'client_secret': enterprise_config.get_credential('client_secret', org_id), } # get the private key key_path = enterprise_config.get_string('priv_key_path', True) if key_path: data_setting = enterprise_config.has_credential('priv_key_data') if data_setting: raise AssertionException( '%s: cannot specify both "priv_key_path" and "%s"' % (enterprise_config.get_full_scope(), data_setting)) logger.debug('%s: reading private key data from file %s', self.name, key_path) auth_dict['private_key_file'] = key_path else: auth_dict['private_key_data'] = enterprise_config.get_credential( 'priv_key_data', org_id) # this check must come after we fetch all the settings enterprise_config.report_unused_values(logger) # open the connection um_endpoint = "https://" + server_options['host'] + server_options[ 'endpoint'] logger.debug('%s: creating connection for org %s at endpoint %s', self.name, org_id, um_endpoint) try: self.connection = connection = umapi_client.Connection( org_id=org_id, auth_dict=auth_dict, ims_host=ims_host, ims_endpoint_jwt=server_options['ims_endpoint_jwt'], user_management_endpoint=um_endpoint, test_mode=options['test_mode'], user_agent="user-sync/" + APP_VERSION, logger=self.logger, ) except Exception as e: raise AssertionException( "Connection to org %s at endpoint %s failed: %s" % (org_id, um_endpoint, e)) logger.debug('%s: connection established', self.name) # wrap the connection in an action manager self.action_manager = ActionManager(connection, org_id, logger)
def __init__(self, name, caller_options): """ :type name: str :type caller_options: dict """ self.name = 'umapi' + name caller_config = user_sync.config.DictConfig( self.name + ' configuration', caller_options) self.trusted = caller_config.get_bool('trusted', True) if self.trusted is None: self.trusted = False builder = user_sync.config.OptionsBuilder(caller_config) builder.set_string_value('logger_name', self.name) builder.set_bool_value('test_mode', False) options = builder.get_options() server_config = caller_config.get_dict_config('server', True) server_builder = user_sync.config.OptionsBuilder(server_config) server_builder.set_string_value('host', 'usermanagement.adobe.io') server_builder.set_string_value('endpoint', '/v2/usermanagement') server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com') server_builder.set_string_value('ims_endpoint_jwt', '/ims/exchange/jwt') server_builder.set_int_value('timeout', 120) server_builder.set_int_value('retries', 3) server_builder.set_bool_value('ssl_verify', True) options['server'] = server_options = server_builder.get_options() enterprise_config = caller_config.get_dict_config('enterprise') enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config) enterprise_builder.require_string_value('org_id') enterprise_builder.require_string_value('tech_acct') options[ 'enterprise'] = enterprise_options = enterprise_builder.get_options( ) self.options = options self.logger = logger = user_sync.connector.helper.create_logger( options) if server_config: server_config.report_unused_values(logger) logger.debug('UMAPI initialized with options: %s', options) ims_host = server_options['ims_host'] self.org_id = org_id = enterprise_options['org_id'] auth_dict = make_auth_dict(self.name, enterprise_config, org_id, enterprise_options['tech_acct'], logger) # this check must come after we fetch all the settings enterprise_config.report_unused_values(logger) # open the connection um_endpoint = "https://" + server_options['host'] + server_options[ 'endpoint'] logger.debug('%s: creating connection for org %s at endpoint %s', self.name, org_id, um_endpoint) try: self.connection = connection = umapi_client.Connection( org_id=org_id, auth_dict=auth_dict, ims_host=ims_host, ims_endpoint_jwt=server_options['ims_endpoint_jwt'], user_management_endpoint=um_endpoint, test_mode=options['test_mode'], user_agent="user-sync/" + app_version, logger=self.logger, timeout_seconds=float(server_options['timeout']), retry_max_attempts=server_options['retries'] + 1, ssl_verify=server_options['ssl_verify']) except Exception as e: raise AssertionException( "Connection to org %s at endpoint %s failed: %s" % (org_id, um_endpoint, e)) logger.debug('%s: connection established', self.name) # wrap the connection in an action manager self.action_manager = ActionManager(connection, org_id, logger)
def __init__(self, name, caller_options): """ :type name: str :type caller_options: dict """ self.name = 'umapi' + name caller_config = user_sync.config.DictConfig( self.name + ' configuration', caller_options) builder = user_sync.config.OptionsBuilder(caller_config) builder.set_string_value('logger_name', self.name) builder.set_bool_value('test_mode', False) options = builder.get_options() server_config = caller_config.get_dict_config('server', True) server_builder = user_sync.config.OptionsBuilder(server_config) server_builder.set_string_value('host', 'usermanagement.adobe.io') server_builder.set_string_value('endpoint', '/v2/usermanagement') server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com') server_builder.set_string_value('ims_endpoint_jwt', '/ims/exchange/jwt') server_builder.set_int_value('timeout', 120) server_builder.set_int_value('retries', 3) options['server'] = server_options = server_builder.get_options() enterprise_config = caller_config.get_dict_config('enterprise') enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config) enterprise_builder.require_string_value('org_id') enterprise_builder.require_string_value('tech_acct') options[ 'enterprise'] = enterprise_options = enterprise_builder.get_options( ) self.options = options self.logger = logger = user_sync.connector.helper.create_logger( options) if server_config: server_config.report_unused_values(logger) logger.debug('UMAPI initialized with options: %s', options) ims_host = server_options['ims_host'] self.org_id = org_id = enterprise_options['org_id'] auth_dict = { 'org_id': org_id, 'tech_acct_id': enterprise_options['tech_acct'], 'api_key': enterprise_config.get_credential('api_key', org_id), 'client_secret': enterprise_config.get_credential('client_secret', org_id), } # get the private key key_path = enterprise_config.get_string('priv_key_path', True) if key_path: data_setting = enterprise_config.has_credential('priv_key_data') if data_setting: raise AssertionException( '%s: cannot specify both "priv_key_path" and "%s"' % (enterprise_config.get_full_scope(), data_setting)) logger.debug('%s: reading private key data from file %s', self.name, key_path) try: with open(key_path, 'r') as f: key_data = f.read() except IOError as e: raise AssertionException( '%s: cannot read file "%s": %s' % (enterprise_config.get_full_scope(), key_path, e)) else: key_data = enterprise_config.get_credential( 'priv_key_data', org_id) # decrypt the private key, if needed passphrase = enterprise_config.get_credential('priv_key_pass', org_id, True) if passphrase: try: key_data = str( RSA.importKey( key_data, passphrase=passphrase).exportKey().decode('ascii')) except (ValueError, IndexError, TypeError) as e: raise AssertionException( '%s: Error decrypting private key, either the password is wrong or: %s' % (enterprise_config.get_full_scope(), e)) auth_dict['private_key_data'] = key_data # this check must come after we fetch all the settings enterprise_config.report_unused_values(logger) # open the connection um_endpoint = "https://" + server_options['host'] + server_options[ 'endpoint'] logger.debug('%s: creating connection for org %s at endpoint %s', self.name, org_id, um_endpoint) try: self.connection = connection = umapi_client.Connection( org_id=org_id, auth_dict=auth_dict, ims_host=ims_host, ims_endpoint_jwt=server_options['ims_endpoint_jwt'], user_management_endpoint=um_endpoint, test_mode=options['test_mode'], user_agent="user-sync/" + app_version, logger=self.logger, timeout_seconds=float(server_options['timeout']), retry_max_attempts=server_options['retries'] + 1, ) except Exception as e: raise AssertionException( "Connection to org %s at endpoint %s failed: %s" % (org_id, um_endpoint, e)) logger.debug('%s: connection established', self.name) # wrap the connection in an action manager self.action_manager = ActionManager(connection, org_id, logger)
def __init__(self, caller_options): caller_config = user_sync.config.DictConfig('<%s configuration>' % self.name, caller_options) builder = user_sync.config.OptionsBuilder(caller_config) # Let just ignore this builder.set_string_value('user_identity_type', None) builder.set_string_value('identity_type_filter', 'all') options = builder.get_options() if not options['identity_type_filter'] == 'all': try: options['identity_type_filter'] = parse_identity_type(options['identity_type_filter']) except Exception as e: raise AssertionException("Error parsing identity_type_filter option: %s" % e) self.filter_by_identity_type = options['identity_type_filter'] server_config = caller_config.get_dict_config('server', True) server_builder = user_sync.config.OptionsBuilder(server_config) server_builder.set_string_value('host', 'usermanagement.adobe.io') server_builder.set_string_value('endpoint', '/v2/usermanagement') server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com') server_builder.set_string_value('ims_endpoint_jwt', '/ims/exchange/jwt') server_builder.set_int_value('timeout', 120) server_builder.set_int_value('retries', 3) options['server'] = server_options = server_builder.get_options() enterprise_config = caller_config.get_dict_config('integration') integration_builder = user_sync.config.OptionsBuilder(enterprise_config) integration_builder.require_string_value('org_id') integration_builder.require_string_value('tech_acct') options['integration'] = integration_options = integration_builder.get_options() self.logger = logger = user_sync.connector.helper.create_logger(options) logger.debug('%s initialized with options: %s', self.name, options) self.options = options ims_host = server_options['ims_host'] self.org_id = org_id = integration_options['org_id'] auth_dict = make_auth_dict(self.name, enterprise_config, org_id, integration_options['tech_acct'], logger) # this check must come after we fetch all the settings caller_config.report_unused_values(logger) # open the connection um_endpoint = "https://" + server_options['host'] + server_options['endpoint'] logger.debug('%s: creating connection for org %s at endpoint %s', self.name, org_id, um_endpoint) try: self.connection = umapi_client.Connection( org_id=org_id, auth_dict=auth_dict, ims_host=ims_host, ims_endpoint_jwt=server_options['ims_endpoint_jwt'], user_management_endpoint=um_endpoint, test_mode=False, user_agent="user-sync/" + app_version, logger=self.logger, timeout_seconds=float(server_options['timeout']), retry_max_attempts=server_options['retries'] + 1, ) except Exception as e: raise AssertionException("Connection to org %s at endpoint %s failed: %s" % (org_id, um_endpoint, e)) logger.debug('%s: connection established', self.name) self.umapi_users = [] self.user_by_usr_key = {}
def begin_work(config_loader): """ :type config_loader: user_sync.config.ConfigLoader """ directory_groups = config_loader.get_directory_groups() rule_config = config_loader.get_rule_options() # make sure that all the adobe groups are from known umapi connector names primary_umapi_config, secondary_umapi_configs = config_loader.get_umapi_options( ) referenced_umapi_names = set() for groups in six.itervalues(directory_groups): for group in groups: umapi_name = group.umapi_name if umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME: referenced_umapi_names.add(umapi_name) referenced_umapi_names.difference_update( six.iterkeys(secondary_umapi_configs)) if len(referenced_umapi_names) > 0: raise AssertionException( 'Adobe groups reference unknown umapi connectors: %s' % referenced_umapi_names) directory_connector = None directory_connector_options = None directory_connector_module_name = config_loader.get_directory_connector_module_name( ) if directory_connector_module_name is not None: directory_connector_module = __import__( directory_connector_module_name, fromlist=['']) directory_connector = user_sync.connector.directory.DirectoryConnector( directory_connector_module) directory_connector_options = config_loader.get_directory_connector_options( directory_connector.name) post_sync_manager = None # get post-sync config unconditionally so we don't get an 'unused key' error post_sync_config = config_loader.get_post_sync_options() if rule_config['strategy'] == 'sync': if post_sync_config: post_sync_manager = PostSyncManager(post_sync_config, rule_config['test_mode']) rule_config[ 'extended_attributes'] |= post_sync_manager.get_directory_attributes( ) else: logger.warn('Post-Sync Connectors only support "sync" strategy') config_loader.check_unused_config_keys() if directory_connector is not None and directory_connector_options is not None: # specify the default user_identity_type if it's not already specified in the options if 'user_identity_type' not in directory_connector_options: directory_connector_options['user_identity_type'] = rule_config[ 'new_account_type'] directory_connector.initialize(directory_connector_options) additional_group_filters = None additional_groups = rule_config.get('additional_groups', None) if additional_groups and isinstance(additional_groups, list): additional_group_filters = [r['source'] for r in additional_groups] if directory_connector is not None: directory_connector.state.additional_group_filters = additional_group_filters # show error dynamic mappings enabled but 'dynamic_group_member_attribute' is not defined if additional_group_filters and directory_connector.state.options[ 'dynamic_group_member_attribute'] is None: raise AssertionException( "Failed to enable dynamic group mappings. 'dynamic_group_member_attribute' is not defined in config" ) primary_name = '.primary' if secondary_umapi_configs else '' umapi_primary_connector = user_sync.connector.umapi.UmapiConnector( primary_name, primary_umapi_config) umapi_other_connectors = {} for secondary_umapi_name, secondary_config in six.iteritems( secondary_umapi_configs): umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector( ".secondary.%s" % secondary_umapi_name, secondary_config) umapi_other_connectors[secondary_umapi_name] = umapi_secondary_conector umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector, umapi_other_connectors) rule_processor = user_sync.rules.RuleProcessor(rule_config) if len(directory_groups) == 0 and rule_processor.will_process_groups(): logger.warning( 'No group mapping specified in configuration but --process-groups requested on command line' ) rule_processor.run(directory_groups, directory_connector, umapi_connectors) # Post sync section if post_sync_manager: post_sync_manager.run(rule_processor.post_sync_data)
def __init__(self, caller_options): caller_config = user_sync.config.DictConfig( '%s configuration' % self.name, caller_options) builder = user_sync.config.OptionsBuilder(caller_config) builder.set_string_value( 'group_filter_format', six.text_type( '(&(|(objectCategory=group)(objectClass=groupOfNames)(objectClass=posixGroup))(cn={group}))' )) builder.set_string_value( 'all_users_filter', six.text_type( '(&(objectClass=user)(objectCategory=person)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))' )) builder.set_string_value('group_member_filter_format', None) builder.set_bool_value('require_tls_cert', False) builder.set_dict_value('two_steps_lookup', None) builder.set_string_value('string_encoding', 'utf8') builder.set_string_value('user_identity_type_format', None) builder.set_string_value('user_email_format', six.text_type('{mail}')) builder.set_string_value('user_username_format', None) builder.set_string_value('user_domain_format', None) builder.set_string_value('user_given_name_format', six.text_type('{givenName}')) builder.set_string_value('user_surname_format', six.text_type('{sn}')) builder.set_string_value('user_country_code_format', six.text_type('{c}')) builder.set_string_value('user_identity_type', None) builder.set_int_value('search_page_size', 200) builder.set_string_value('logger_name', LDAPDirectoryConnector.name) host = builder.require_string_value('host') username = builder.require_string_value('username') builder.require_string_value('base_dn') options = builder.get_options() if options['two_steps_lookup'] is not None: ts_config = caller_config.get_dict_config('two_steps_lookup', True) ts_builder = user_sync.config.OptionsBuilder(ts_config) ts_builder.require_string_value('group_member_attribute_name') ts_builder.set_bool_value('nested_group', False) self.two_steps_lookup = True options['two_steps_lookup'] = ts_builder.get_options() if options['group_member_filter_format']: raise AssertionException( "Cannot define both 'group_member_attribute_name' and 'group_member_filter_format' in config" ) else: self.two_steps_lookup = False if not options['group_member_filter_format']: options['group_member_filter_format'] = six.text_type( '(memberOf={group_dn})') self.options = options self.logger = logger = user_sync.connector.helper.create_logger( options) logger.debug('%s initialized with options: %s', self.name, options) LDAPValueFormatter.encoding = options['string_encoding'] self.user_identity_type = user_sync.identity_type.parse_identity_type( options['user_identity_type']) self.user_identity_type_formatter = LDAPValueFormatter( options['user_identity_type_format']) self.user_email_formatter = LDAPValueFormatter( options['user_email_format']) self.user_username_formatter = LDAPValueFormatter( options['user_username_format']) self.user_domain_formatter = LDAPValueFormatter( options['user_domain_format']) self.user_given_name_formatter = LDAPValueFormatter( options['user_given_name_format']) self.user_surname_formatter = LDAPValueFormatter( options['user_surname_format']) self.user_country_code_formatter = LDAPValueFormatter( options['user_country_code_format']) password = caller_config.get_credential('password', options['username']) # this check must come after we get the password value caller_config.report_unused_values(logger) logger.debug('Connecting to: %s using username: %s', host, username) if not options['require_tls_cert']: ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER) try: # Be careful in Py2!! We are setting bytes_mode = False, so we must give all attribute names # and other protocol-defined strings (such as username) as Unicode. But the PyYAML parser # will always return ascii strings as str type (rather than Unicode). So we must be careful # to upconvert all parameter strings to unicode when passing them in. connection = ldap.initialize(host, bytes_mode=False) connection.protocol_version = ldap.VERSION3 connection.set_option(ldap.OPT_REFERRALS, 0) connection.simple_bind_s(six.text_type(username), six.text_type(password)) except Exception as e: raise AssertionException('LDAP connection failure: %s' % e) self.connection = connection logger.debug('Connected') self.user_by_dn = {} self.additional_group_filters = None