def insert_updated_entries(data_dict, when_changed, data_type):
"""Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer."""
insertion_counter = 0
conn = connect_to_db()
for entry in data_dict:
if entry.whenChanged.value > when_changed:
if data_type == "user":
standardized_entry = inbound_user_filter(entry, "ldap")
else:
standardized_entry = inbound_group_filter(entry, "ldap")
entry_modified_timestamp = entry.whenChanged.value.strftime(
"%Y-%m-%dT%H:%M:%S.%f+00:00")
inbound_entry = {
"data": standardized_entry,
"data_type": data_type,
"sync_type": "delta",
"timestamp": entry_modified_timestamp,
"provider_id": LDAP_DC,
}
LOGGER.debug(
"Inserting LDAP %s into inbound queue: %s",
data_type,
standardized_entry["remote_id"],
)
r.table("inbound_queue").insert(inbound_entry).run(conn)
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, "delta",
entry_modified_timestamp)
insertion_counter += 1
conn.close()
LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def fetch_ldap_data(data_type):
"""
Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync,
inserts data into RethinkDB, and initiates a new thread for a delta sync for data_type.
"""
connect_to_db()
if data_type == "user":
search_filter = "(objectClass=person)"
elif data_type == "group":
search_filter = "(objectClass=group)"
server = Server(LDAP_SERVER, get_info=ALL)
conn = Connection(server, user=LDAP_USER, password=LDAP_PASS)
if not conn.bind():
LOGGER.error("Error connecting to LDAP server %s : %s", LDAP_SERVER,
conn.result)
conn.search(
search_base=LDAP_DC,
search_filter=search_filter,
attributes=ldap3.ALL_ATTRIBUTES,
)
insert_to_db(data_dict=conn.entries, data_type=data_type)
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, "initial")
def insert_to_db(data_dict, when_changed):
"""Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer."""
insertion_counter = 0
for entry in data_dict:
if entry.whenChanged.value > when_changed:
entry_data = json.loads(entry.entry_to_json())["attributes"]
if "person" in entry.objectClass.value:
data_type = "user"
standardized_entry = inbound_user_filter(entry_data, "ldap")
else:
data_type = "group"
standardized_entry = inbound_group_filter(entry_data, "ldap")
entry_modified_timestamp = entry.whenChanged.value.strftime(
"%Y-%m-%dT%H:%M:%S.%f+00:00")
inbound_entry = {
"data": standardized_entry,
"data_type": data_type,
"sync_type": "delta",
"timestamp": entry_modified_timestamp,
"provider_id": LDAP_DC,
}
r.table("inbound_queue").insert(inbound_entry).run()
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, "delta",
entry_modified_timestamp)
insertion_counter += 1
LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def initialize_aad_sync():
"""Initialize a sync with Azure Active Directory."""
LOGGER.info("connecting to RethinkDB...")
connect_to_db()
LOGGER.info("Successfully connected to RethinkDB!")
provider_id = TENANT_ID
db_user_payload = check_last_sync("azure-user", "initial")
if not db_user_payload:
LOGGER.info(
"No initial AAD user sync was found. Starting initial AAD user sync now."
)
LOGGER.info("Getting Users...")
users = fetch_users()
if users:
insert_user_to_db(users)
while "@odata.nextLink" in users:
users = fetch_next_payload(users["@odata.nextLink"])
if users:
insert_user_to_db(users)
else:
break
save_sync_time(provider_id, "azure-user", "initial")
LOGGER.info("Initial user upload complete :)")
else:
LOGGER.info(
"An error occurred when uploading users. Please check the logs."
)
db_group_payload = check_last_sync("azure-group", "initial")
if not db_group_payload:
LOGGER.info(
"No initial AAD group sync was found. Starting initial AAD group sync now."
)
LOGGER.info("Getting Groups with Members...")
groups = fetch_groups_with_members()
if groups:
insert_group_to_db(groups)
while "@odata.nextLink" in groups:
groups = fetch_next_payload(groups["@odata.nextLink"])
if groups:
insert_group_to_db(groups)
else:
break
save_sync_time(provider_id, "azure-group", "initial")
LOGGER.info("Initial group upload complete :)")
else:
LOGGER.info(
"An error occurred when uploading groups. Please check the logs."
)
if db_group_payload and db_user_payload:
LOGGER.info("The initial sync has already been run.")
def fetch_ldap_data(data_type):
"""
Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync,
inserts data into RethinkDB, and initiates a new thread for a delta sync for data_type.
"""
if data_type == "user":
search_filter = "(objectClass=person)"
search_base = USER_BASE_DN
elif data_type == "group":
search_filter = "(objectClass=group)"
search_base = GROUP_BASE_DN
ldap_connection = ldap_connector.await_connection(LDAP_SERVER, LDAP_USER,
LDAP_PASS)
search_parameters = {
"search_base": search_base,
"search_filter": search_filter,
"attributes": ldap3.ALL_ATTRIBUTES,
"paged_size": LDAP_SEARCH_PAGE_SIZE,
}
entry_count = 0
LOGGER.info("Importing %ss..", data_type)
conn = connect_to_db()
while True:
start_time = time.clock()
ldap_connection.search(**search_parameters)
record_count = len(ldap_connection.entries)
LOGGER.info(
"Got %s entries in %s seconds.",
record_count,
"%.3f" % (time.clock() - start_time),
)
for entry in ldap_connection.entries:
entry_count = entry_count + 1
insert_to_db(entry=entry, data_type=data_type, conn=conn)
# 1.2.840.113556.1.4.319 is the OID/extended control for PagedResults
cookie = ldap_connection.result["controls"]["1.2.840.113556.1.4.319"][
"value"]["cookie"]
if cookie:
search_parameters["paged_cookie"] = cookie
else:
LOGGER.info("Imported %s entries from Active Directory",
entry_count)
break
sync_source = "ldap-" + data_type
save_sync_time(LDAP_DC, sync_source, "initial", conn)
conn.close()
def insert_to_db(data_dict, when_changed, data_type):
"""Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer."""
insertion_counter = 0
conn = connect_to_db()
for entry in data_dict:
entry_to_insert = {}
entry_json = json.loads(entry.entry_to_json())
entry_attributes = entry_json["attributes"]
for attribute in entry_attributes:
if len(entry_attributes[attribute]) > 1:
entry_to_insert[attribute] = entry_attributes[attribute]
else:
entry_to_insert[attribute] = entry_attributes[attribute][0]
if entry.whenChanged.value > when_changed:
if data_type == "user":
standardized_entry = inbound_user_filter(
entry_to_insert, "ldap")
else:
standardized_entry = inbound_group_filter(
entry_to_insert, "ldap")
entry_modified_timestamp = entry.whenChanged.value.strftime(
"%Y-%m-%dT%H:%M:%S.%f+00:00")
inbound_entry = {
"data": standardized_entry,
"data_type": data_type,
"sync_type": "delta",
"timestamp": entry_modified_timestamp,
"provider_id": LDAP_DC,
}
add_transaction(inbound_entry)
r.table("inbound_queue").insert(inbound_entry).run(conn)
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, "delta",
entry_modified_timestamp)
insertion_counter += 1
conn.close()
LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def inbound_sync_listener():
"""Initialize a delta inbound sync with Azure Active Directory."""
while True: # pylint: disable=too-many-nested-blocks
provider_id = TENANT_ID
try:
initial_sync_time = check_last_sync("azure-user", "initial")
LOGGER.info(initial_sync_time)
LOGGER.info("This is your initial sync time")
initial_sync_time = initial_sync_time["timestamp"][:26]
latest_delta_sync_time = get_last_delta_sync(provider_id, "delta")
if latest_delta_sync_time:
latest_delta_sync_time = latest_delta_sync_time[
"timestamp"][:26]
previous_sync_datetime = datetime.strptime(
latest_delta_sync_time, "%Y-%m-%dT%H:%M:%S.%f")
else:
previous_sync_datetime = datetime.strptime(
initial_sync_time, "%Y-%m-%dT%H:%M:%S.%f")
# Create an eventhub client.
LOGGER.info(ADDRESS)
client = EventHubClient(ADDRESS,
debug=False,
username=USER,
password=KEY)
try:
LOGGER.info("Opening connection to EventHub...")
# Set prefetch to 1, we only want one event at a time.
receiver = client.add_receiver(CONSUMER_GROUP,
PARTITION,
prefetch=1,
offset=OFFSET)
# Open the connection to the EventHub.
client.run()
# Get one event from EventHub.
batch = receiver.receive(timeout=5000)
while batch:
for event_data in batch:
# Get the event as a json record from the batch of events.
event_json = event_data.body_as_json()
record = event_json["records"][0]
operation_name = record["operationName"]
time = record["time"][:26]
record_timestamp = datetime.strptime(
time, "%Y-%m-%dT%H:%M:%S.%f")
# Only process events logged after the previous initial/delta sync.
# Only grab events concerning User or Group objects.
if (operation_name in VALID_OPERATIONS
and record_timestamp > previous_sync_datetime):
data = {
"initated_by":
record["properties"]["initiatedBy"],
"target_resources":
record["properties"]["targetResources"],
"operation_name":
operation_name,
"resultType":
record["resultType"],
}
LOGGER.info("Operation name: %s", operation_name)
LOGGER.info("Record to Change: %s", record)
record_timestamp_utc = record_timestamp.isoformat()
insert_change_to_db(data, record_timestamp_utc)
sync_source = "azure-" + VALID_OPERATIONS[
operation_name]
provider_id = TENANT_ID
conn = connect_to_db()
save_sync_time(
provider_id,
sync_source,
"delta",
conn,
record_timestamp_utc,
)
conn.close()
previous_sync_datetime = record_timestamp
batch = receiver.receive(timeout=50)
LOGGER.info("Closing connection to EventHub...")
# Close the connection to the EventHub.
client.stop()
except KeyboardInterrupt:
pass
finally:
client.stop()
except ExpectedError as err:
LOGGER.debug((
"%s Repolling after %s seconds...",
err.__str__,
LISTENER_POLLING_DELAY,
))
time.sleep(LISTENER_POLLING_DELAY)
except Exception as err:
LOGGER.exception(err)
raise err
