def insert_updated_entries(data_dict, when_changed, data_type): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" insertion_counter = 0 conn = connect_to_db() for entry in data_dict: if entry.whenChanged.value > when_changed: if data_type == "user": standardized_entry = inbound_user_filter(entry, "ldap") else: standardized_entry = inbound_group_filter(entry, "ldap") entry_modified_timestamp = entry.whenChanged.value.strftime( "%Y-%m-%dT%H:%M:%S.%f+00:00") inbound_entry = { "data": standardized_entry, "data_type": data_type, "sync_type": "delta", "timestamp": entry_modified_timestamp, "provider_id": LDAP_DC, } LOGGER.debug( "Inserting LDAP %s into inbound queue: %s", data_type, standardized_entry["remote_id"], ) r.table("inbound_queue").insert(inbound_entry).run(conn) sync_source = "ldap-" + data_type provider_id = LDAP_DC save_sync_time(provider_id, sync_source, "delta", entry_modified_timestamp) insertion_counter += 1 conn.close() LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def fetch_ldap_data(data_type): """ Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync, inserts data into RethinkDB, and initiates a new thread for a delta sync for data_type. """ connect_to_db() if data_type == "user": search_filter = "(objectClass=person)" elif data_type == "group": search_filter = "(objectClass=group)" server = Server(LDAP_SERVER, get_info=ALL) conn = Connection(server, user=LDAP_USER, password=LDAP_PASS) if not conn.bind(): LOGGER.error("Error connecting to LDAP server %s : %s", LDAP_SERVER, conn.result) conn.search( search_base=LDAP_DC, search_filter=search_filter, attributes=ldap3.ALL_ATTRIBUTES, ) insert_to_db(data_dict=conn.entries, data_type=data_type) sync_source = "ldap-" + data_type provider_id = LDAP_DC save_sync_time(provider_id, sync_source, "initial")
def insert_to_db(data_dict, when_changed): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" insertion_counter = 0 for entry in data_dict: if entry.whenChanged.value > when_changed: entry_data = json.loads(entry.entry_to_json())["attributes"] if "person" in entry.objectClass.value: data_type = "user" standardized_entry = inbound_user_filter(entry_data, "ldap") else: data_type = "group" standardized_entry = inbound_group_filter(entry_data, "ldap") entry_modified_timestamp = entry.whenChanged.value.strftime( "%Y-%m-%dT%H:%M:%S.%f+00:00") inbound_entry = { "data": standardized_entry, "data_type": data_type, "sync_type": "delta", "timestamp": entry_modified_timestamp, "provider_id": LDAP_DC, } r.table("inbound_queue").insert(inbound_entry).run() sync_source = "ldap-" + data_type provider_id = LDAP_DC save_sync_time(provider_id, sync_source, "delta", entry_modified_timestamp) insertion_counter += 1 LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def initialize_aad_sync(): """Initialize a sync with Azure Active Directory.""" LOGGER.info("connecting to RethinkDB...") connect_to_db() LOGGER.info("Successfully connected to RethinkDB!") provider_id = TENANT_ID db_user_payload = check_last_sync("azure-user", "initial") if not db_user_payload: LOGGER.info( "No initial AAD user sync was found. Starting initial AAD user sync now." ) LOGGER.info("Getting Users...") users = fetch_users() if users: insert_user_to_db(users) while "@odata.nextLink" in users: users = fetch_next_payload(users["@odata.nextLink"]) if users: insert_user_to_db(users) else: break save_sync_time(provider_id, "azure-user", "initial") LOGGER.info("Initial user upload complete :)") else: LOGGER.info( "An error occurred when uploading users. Please check the logs." ) db_group_payload = check_last_sync("azure-group", "initial") if not db_group_payload: LOGGER.info( "No initial AAD group sync was found. Starting initial AAD group sync now." ) LOGGER.info("Getting Groups with Members...") groups = fetch_groups_with_members() if groups: insert_group_to_db(groups) while "@odata.nextLink" in groups: groups = fetch_next_payload(groups["@odata.nextLink"]) if groups: insert_group_to_db(groups) else: break save_sync_time(provider_id, "azure-group", "initial") LOGGER.info("Initial group upload complete :)") else: LOGGER.info( "An error occurred when uploading groups. Please check the logs." ) if db_group_payload and db_user_payload: LOGGER.info("The initial sync has already been run.")
def fetch_ldap_data(data_type): """ Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync, inserts data into RethinkDB, and initiates a new thread for a delta sync for data_type. """ if data_type == "user": search_filter = "(objectClass=person)" search_base = USER_BASE_DN elif data_type == "group": search_filter = "(objectClass=group)" search_base = GROUP_BASE_DN ldap_connection = ldap_connector.await_connection(LDAP_SERVER, LDAP_USER, LDAP_PASS) search_parameters = { "search_base": search_base, "search_filter": search_filter, "attributes": ldap3.ALL_ATTRIBUTES, "paged_size": LDAP_SEARCH_PAGE_SIZE, } entry_count = 0 LOGGER.info("Importing %ss..", data_type) conn = connect_to_db() while True: start_time = time.clock() ldap_connection.search(**search_parameters) record_count = len(ldap_connection.entries) LOGGER.info( "Got %s entries in %s seconds.", record_count, "%.3f" % (time.clock() - start_time), ) for entry in ldap_connection.entries: entry_count = entry_count + 1 insert_to_db(entry=entry, data_type=data_type, conn=conn) # 1.2.840.113556.1.4.319 is the OID/extended control for PagedResults cookie = ldap_connection.result["controls"]["1.2.840.113556.1.4.319"][ "value"]["cookie"] if cookie: search_parameters["paged_cookie"] = cookie else: LOGGER.info("Imported %s entries from Active Directory", entry_count) break sync_source = "ldap-" + data_type save_sync_time(LDAP_DC, sync_source, "initial", conn) conn.close()
def insert_to_db(data_dict, when_changed, data_type): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" insertion_counter = 0 conn = connect_to_db() for entry in data_dict: entry_to_insert = {} entry_json = json.loads(entry.entry_to_json()) entry_attributes = entry_json["attributes"] for attribute in entry_attributes: if len(entry_attributes[attribute]) > 1: entry_to_insert[attribute] = entry_attributes[attribute] else: entry_to_insert[attribute] = entry_attributes[attribute][0] if entry.whenChanged.value > when_changed: if data_type == "user": standardized_entry = inbound_user_filter( entry_to_insert, "ldap") else: standardized_entry = inbound_group_filter( entry_to_insert, "ldap") entry_modified_timestamp = entry.whenChanged.value.strftime( "%Y-%m-%dT%H:%M:%S.%f+00:00") inbound_entry = { "data": standardized_entry, "data_type": data_type, "sync_type": "delta", "timestamp": entry_modified_timestamp, "provider_id": LDAP_DC, } add_transaction(inbound_entry) r.table("inbound_queue").insert(inbound_entry).run(conn) sync_source = "ldap-" + data_type provider_id = LDAP_DC save_sync_time(provider_id, sync_source, "delta", entry_modified_timestamp) insertion_counter += 1 conn.close() LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def inbound_sync_listener(): """Initialize a delta inbound sync with Azure Active Directory.""" while True: # pylint: disable=too-many-nested-blocks provider_id = TENANT_ID try: initial_sync_time = check_last_sync("azure-user", "initial") LOGGER.info(initial_sync_time) LOGGER.info("This is your initial sync time") initial_sync_time = initial_sync_time["timestamp"][:26] latest_delta_sync_time = get_last_delta_sync(provider_id, "delta") if latest_delta_sync_time: latest_delta_sync_time = latest_delta_sync_time[ "timestamp"][:26] previous_sync_datetime = datetime.strptime( latest_delta_sync_time, "%Y-%m-%dT%H:%M:%S.%f") else: previous_sync_datetime = datetime.strptime( initial_sync_time, "%Y-%m-%dT%H:%M:%S.%f") # Create an eventhub client. LOGGER.info(ADDRESS) client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) try: LOGGER.info("Opening connection to EventHub...") # Set prefetch to 1, we only want one event at a time. receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=1, offset=OFFSET) # Open the connection to the EventHub. client.run() # Get one event from EventHub. batch = receiver.receive(timeout=5000) while batch: for event_data in batch: # Get the event as a json record from the batch of events. event_json = event_data.body_as_json() record = event_json["records"][0] operation_name = record["operationName"] time = record["time"][:26] record_timestamp = datetime.strptime( time, "%Y-%m-%dT%H:%M:%S.%f") # Only process events logged after the previous initial/delta sync. # Only grab events concerning User or Group objects. if (operation_name in VALID_OPERATIONS and record_timestamp > previous_sync_datetime): data = { "initated_by": record["properties"]["initiatedBy"], "target_resources": record["properties"]["targetResources"], "operation_name": operation_name, "resultType": record["resultType"], } LOGGER.info("Operation name: %s", operation_name) LOGGER.info("Record to Change: %s", record) record_timestamp_utc = record_timestamp.isoformat() insert_change_to_db(data, record_timestamp_utc) sync_source = "azure-" + VALID_OPERATIONS[ operation_name] provider_id = TENANT_ID conn = connect_to_db() save_sync_time( provider_id, sync_source, "delta", conn, record_timestamp_utc, ) conn.close() previous_sync_datetime = record_timestamp batch = receiver.receive(timeout=50) LOGGER.info("Closing connection to EventHub...") # Close the connection to the EventHub. client.stop() except KeyboardInterrupt: pass finally: client.stop() except ExpectedError as err: LOGGER.debug(( "%s Repolling after %s seconds...", err.__str__, LISTENER_POLLING_DELAY, )) time.sleep(LISTENER_POLLING_DELAY) except Exception as err: LOGGER.exception(err) raise err