def fetch_ldap_data(data_type):
"""
Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync,
inserts data into RethinkDB, and initiates a new thread for a delta sync for data_type.
"""
connect_to_db()
if data_type == "user":
search_filter = LDAP_FILTER_USER
elif data_type == "group":
search_filter = LDAP_FILTER_GROUP
server = Server(LDAP_SERVER, get_info=ALL)
conn = Connection(server, user=LDAP_USER, password=LDAP_PASS)
if not conn.bind():
LOGGER.error(
"Error connecting to LDAP server %s : %s", LDAP_SERVER, conn.result
)
conn.search(
search_base=LDAP_DC,
search_filter=search_filter,
attributes=ldap3.ALL_ATTRIBUTES,
)
insert_to_db(data_dict=conn.entries, data_type=data_type)
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, "initial")
def insert_to_db(data_dict, when_changed):
"""Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer."""
insertion_counter = 0
for entry in data_dict:
if entry.whenChanged.value > when_changed:
entry_data = json.loads(entry.entry_to_json())["attributes"]
if "person" in entry.objectClass.value:
data_type = "user"
standardized_entry = inbound_user_filter(entry_data, "ldap")
else:
data_type = "group"
standardized_entry = inbound_group_filter(entry_data, "ldap")
entry_modified_timestamp = entry.whenChanged.value.strftime(
"%Y-%m-%dT%H:%M:%S.%f+00:00")
# TODO: Decide on inbound_entry format
inbound_entry = {
"data": standardized_entry,
"data_type": data_type,
"sync_type": "delta",
"timestamp": entry_modified_timestamp,
"provider_id": LDAP_DC,
}
r.table("inbound_queue").insert(inbound_entry).run()
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, "delta",
entry_modified_timestamp)
insertion_counter += 1
LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def initialize_aad_sync():
"""Initialize a sync with Azure Active Directory."""
LOGGER.info("connecting to RethinkDB...")
connect_to_db()
LOGGER.info("Successfully connected to RethinkDB!")
provider_id = TENANT_ID
db_user_payload = check_last_sync("azure-user", "initial")
if not db_user_payload:
LOGGER.info(
"No initial AAD user sync was found. Starting initial AAD user sync now."
)
LOGGER.info("Getting Users...")
users = fetch_users()
if users:
insert_user_to_db(users)
while "@odata.nextLink" in users:
users = fetch_next_payload(users["@odata.nextLink"])
if users:
insert_user_to_db(users)
else:
break
save_sync_time(provider_id, "azure-user", "initial")
LOGGER.info("Initial user upload complete :)")
else:
LOGGER.info(
"An error occurred when uploading users. Please check the logs."
)
db_group_payload = check_last_sync("azure-group", "initial")
if not db_group_payload:
LOGGER.info(
"No initial AAD group sync was found. Starting initial AAD group sync now."
)
LOGGER.info("Getting Groups with Members...")
groups = fetch_groups_with_members()
if groups:
insert_group_to_db(groups)
while "@odata.nextLink" in groups:
groups = fetch_next_payload(groups["@odata.nextLink"])
if groups:
insert_group_to_db(groups)
else:
break
save_sync_time(provider_id, "azure-group", "initial")
LOGGER.info("Initial group upload complete :)")
else:
LOGGER.info(
"An error occurred when uploading groups. Please check the logs."
)
if db_group_payload and db_user_payload:
LOGGER.info("The initial sync has already been run.")
def fetch_ldap_data(sync_type, data_type):
"""
Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync,
and inserts data into RethinkDB.
"""
connect_to_db()
if sync_type == "delta":
last_sync = (
r.table("sync_tracker")
.filter({"source": "ldap-" + data_type})
.coerce_to("array")
.run()
)
last_sync_time = ldap_payload_transformer.to_date_ldap_query(
rethink_timestamp=last_sync[0]["timestamp"]
)
if data_type == "user":
search_filter = LDAP_FILTER_USER_DELTA % last_sync_time
elif data_type == "group":
search_filter = LDAP_FILTER_GROUP_DELTA % last_sync_time
elif sync_type == "initial":
if data_type == "user":
search_filter = LDAP_FILTER_USER
elif data_type == "group":
search_filter = LDAP_FILTER_GROUP
server = Server(LDAP_SERVER, get_info=ALL)
conn = Connection(server, user=LDAP_USER, password=LDAP_PASS)
if not conn.bind():
LOGGER.error(
"Error connecting to LDAP server %s : %s", LDAP_SERVER, conn.result
)
conn.search(
search_base=LDAP_DC,
search_filter=search_filter,
attributes=ldap3.ALL_ATTRIBUTES,
)
insert_to_db(data_dict=conn.entries, data_type=data_type)
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, sync_type)
def inbound_sync_listener():
"""Initialize a delta inbound sync with Azure Active Directory."""
while True: # pylint: disable=too-many-nested-blocks
provider_id = TENANT_ID
try:
LOGGER.info("Connecting to RethinkDB...")
connect_to_db()
LOGGER.info("Successfully connected to RethinkDB!")
initial_sync_time = check_last_sync("azure-user", "initial")
initial_sync_time = initial_sync_time[0]["timestamp"][:26]
latest_delta_sync_time = get_last_delta_sync(provider_id, "delta")
if latest_delta_sync_time:
latest_delta_sync_time = latest_delta_sync_time[
"timestamp"][:26]
previous_sync_datetime = datetime.strptime(
latest_delta_sync_time, "%Y-%m-%dT%H:%M:%S.%f")
else:
previous_sync_datetime = datetime.strptime(
initial_sync_time, "%Y-%m-%dT%H:%M:%S.%f")
# Create an eventhub client.
client = EventHubClient(ADDRESS,
debug=False,
username=USER,
password=KEY)
try:
LOGGER.info("Opening connection to EventHub...")
# Set prefetch to 1, we only want one event at a time.
receiver = client.add_receiver(CONSUMER_GROUP,
PARTITION,
prefetch=1,
offset=OFFSET)
# Open the connection to the EventHub.
client.run()
# Get one event from EventHub.
batch = receiver.receive(timeout=5000)
while batch:
for event_data in batch:
# Get the event as a json record from the batch of events.
event_json = event_data.body_as_json()
record = event_json["records"][0]
operation_name = record["operationName"]
time = record["time"][:26]
record_timestamp = datetime.strptime(
time, "%Y-%m-%dT%H:%M:%S.%f")
# Only process events logged after the previous initial/delta sync.
# Only grab events concerning User or Group objects.
if (operation_name in VALID_OPERATIONS
and record_timestamp > previous_sync_datetime):
data = {
"initated_by":
record["properties"]["initiatedBy"],
"target_resources":
record["properties"]["targetResources"],
"operation_name":
operation_name,
"resultType":
record["resultType"],
}
LOGGER.info("Operation name: %s", operation_name)
LOGGER.info("Record to Change: %s", record)
record_timestamp_utc = record_timestamp.isoformat()
insert_change_to_db(data, record_timestamp_utc)
sync_source = "azure-" + VALID_OPERATIONS[
operation_name]
provider_id = TENANT_ID
save_sync_time(provider_id, sync_source, "delta",
record_timestamp_utc)
previous_sync_datetime = record_timestamp
batch = receiver.receive(timeout=50)
LOGGER.info("Closing connection to EventHub...")
# Close the connection to the EventHub.
client.stop()
except KeyboardInterrupt:
pass
finally:
client.stop()
except ExpectedError as err:
LOGGER.debug(
("%s Repolling after %s seconds...", err.__str__, DELAY))
time.sleep(DELAY)
except Exception as err:
LOGGER.exception(err)
raise err