def inbound_sync_listener():
"""Initialize a delta inbound sync between the inbound queue and sawtooth."""
LOGGER.info("Starting inbound sync listener...")
LOGGER.info("Connecting to RethinkDB...")
connect_to_db()
LOGGER.info("Successfully connected to RethinkDB!")
while True:
try:
queue_entry = peek_at_queue(INBOUND_QUEUE)
LOGGER.info(
"Received queue entry %s from outbound queue...", queue_entry["id"]
)
data_type = queue_entry["data_type"]
LOGGER.info("Putting %s into Sawtooth...", data_type)
# TODO: Validate queue_entry.
# TODO: Transform or reject invalid entries.
# TODO: Get queue_entry object from NEXT state table.
# TODO: Update object or create if it doesn't exist.
LOGGER.debug(queue_entry)
LOGGER.info("Putting queue entry into changelog...")
put_entry_changelog(queue_entry, DIRECTION)
LOGGER.info("Deleting queue entry from outbound queue...")
entry_id = queue_entry["id"]
delete_entry_queue(entry_id, INBOUND_QUEUE)
except ExpectedError as err:
time.sleep(DELAY)
except Exception as err:
LOGGER.exception(err)
raise err
def fetch_ldap_data(data_type):
"""
Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync,
inserts data into RethinkDB, and initiates a new thread for a delta sync for data_type.
"""
connect_to_db()
if data_type == "user":
search_filter = LDAP_FILTER_USER
elif data_type == "group":
search_filter = LDAP_FILTER_GROUP
server = Server(LDAP_SERVER, get_info=ALL)
conn = Connection(server, user=LDAP_USER, password=LDAP_PASS)
if not conn.bind():
LOGGER.error(
"Error connecting to LDAP server %s : %s", LDAP_SERVER, conn.result
)
conn.search(
search_base=LDAP_DC,
search_filter=search_filter,
attributes=ldap3.ALL_ATTRIBUTES,
)
insert_to_db(data_dict=conn.entries, data_type=data_type)
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, "initial")
def outbound_sync_listener():
"""Initialize a delta outbound sync with Azure Active Directory."""
LOGGER.info("Starting outbound sync listener...")
LOGGER.info("Connecting to RethinkDB...")
connect_to_db()
LOGGER.info("Successfully connected to RethinkDB!")
while True:
try:
queue_entry = peek_at_queue(OUTBOUND_QUEUE, TENANT_ID)
LOGGER.info("Received queue entry %s from outbound queue...",
queue_entry["id"])
data_type = queue_entry["data_type"]
LOGGER.info("Putting %s into aad...", data_type)
if is_entry_in_aad(queue_entry):
update_entry_aad(queue_entry)
else:
create_entry_aad(queue_entry)
LOGGER.info("Putting queue entry into changelog...")
put_entry_changelog(queue_entry, "outbound")
LOGGER.info("Deleting queue entry from outbound queue...")
entry_id = queue_entry["id"]
delete_entry_queue(entry_id, OUTBOUND_QUEUE)
except ExpectedError as err:
LOGGER.debug(
("%s Repolling after %s seconds...", err.__str__, DELAY))
time.sleep(DELAY)
except Exception as err:
LOGGER.exception(err)
raise err
def fetch_ldap_data():
"""
Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync,
and inserts data into RethinkDB.
"""
LOGGER.debug("Connecting to RethinkDB...")
connect_to_db()
LOGGER.debug("Successfully connected to RethinkDB")
last_sync = (r.table("sync_tracker").filter({
"provider_id": LDAP_DC
}).max("timestamp").coerce_to("object").run())
last_sync_time = last_sync["timestamp"]
last_sync_time_formatted = to_date_ldap_query(
rethink_timestamp=last_sync_time)
search_filter = (
"(&(|(objectClass=person)(objectClass=group))(whenChanged>=%s))" %
last_sync_time_formatted)
server = Server(LDAP_SERVER, get_info=ALL)
ldap_conn = Connection(server, user=LDAP_USER, password=LDAP_PASS)
if not ldap_conn.bind():
LOGGER.error("Error connecting to LDAP server %s : %s", LDAP_SERVER,
ldap_conn.result)
ldap_conn.search(
search_base=LDAP_DC,
search_filter=search_filter,
attributes=ldap3.ALL_ATTRIBUTES,
)
parsed_last_sync_time = datetime.strptime(
last_sync_time.split("+")[0],
"%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=timezone.utc)
insert_to_db(data_dict=ldap_conn.entries,
when_changed=parsed_last_sync_time)
def initialize_aad_sync():
"""Initialize a sync with Azure Active Directory."""
LOGGER.info("connecting to RethinkDB...")
connect_to_db()
LOGGER.info("Successfully connected to RethinkDB!")
provider_id = TENANT_ID
db_user_payload = check_last_sync("azure-user", "initial")
if not db_user_payload:
LOGGER.info(
"No initial AAD user sync was found. Starting initial AAD user sync now."
)
LOGGER.info("Getting Users...")
users = fetch_users()
if users:
insert_user_to_db(users)
while "@odata.nextLink" in users:
users = fetch_next_payload(users["@odata.nextLink"])
if users:
insert_user_to_db(users)
else:
break
save_sync_time(provider_id, "azure-user", "initial")
LOGGER.info("Initial user upload complete :)")
else:
LOGGER.info(
"An error occurred when uploading users. Please check the logs."
)
db_group_payload = check_last_sync("azure-group", "initial")
if not db_group_payload:
LOGGER.info(
"No initial AAD group sync was found. Starting initial AAD group sync now."
)
LOGGER.info("Getting Groups with Members...")
groups = fetch_groups_with_members()
if groups:
insert_group_to_db(groups)
while "@odata.nextLink" in groups:
groups = fetch_next_payload(groups["@odata.nextLink"])
if groups:
insert_group_to_db(groups)
else:
break
save_sync_time(provider_id, "azure-group", "initial")
LOGGER.info("Initial group upload complete :)")
else:
LOGGER.info(
"An error occurred when uploading groups. Please check the logs."
)
if db_group_payload and db_user_payload:
LOGGER.info("The initial sync has already been run.")
def initialize_ldap_sync():
"""
Checks if LDAP initial syncs has been ran. If not, run initial sync for both ldap users
and groups. If initial syncs have been completed, restart the inbound delta syncs.
"""
if LDAP_DC:
connect_to_db()
user_sync_completed = False
group_sync_completed = False
db_user_payload = check_last_sync("ldap-user", "initial")
if not db_user_payload:
LOGGER.info(
"No initial AD user sync was found. Starting initial AD user sync now."
)
LOGGER.info("Getting AD Users...")
fetch_ldap_data(data_type="user")
LOGGER.info("Initial AD user upload completed.")
user_sync_completed = True
db_group_payload = check_last_sync("ldap-group", "initial")
if not db_group_payload:
LOGGER.info(
"No initial AD group sync was found. Starting initial AD group sync now."
)
LOGGER.info("Getting Groups with Members...")
fetch_ldap_data(data_type="group")
LOGGER.info("Initial AD group upload completed.")
group_sync_completed = True
if user_sync_completed and group_sync_completed:
initiate_delta_sync()
else:
LOGGER.info(
"Initial syncs did not complete successfully, LDAP delta sync will not start."
)
if db_user_payload and db_group_payload:
LOGGER.info("The LDAP initial sync has already been run.")
initiate_delta_sync()
else:
LOGGER.info("LDAP Domain Controller is not provided, skipping LDAP sync.")
def process_ldap_outbound():
"""
While there are items in OUTBOUND_QUEUE table, get earliest entry in table.
Check if entry is in ldap and process the entry accordingly. Once processed,
record entry in changelog and delete the entry from the outbound queue.
"""
connect_to_db()
ldap_conn = connect_to_ldap()
while not r.table(OUTBOUND_QUEUE).filter({"provider_id": LDAP_DC}).is_empty().run():
queue_entry = peek_at_queue(table_name=OUTBOUND_QUEUE, provider_id=LDAP_DC)
if is_entry_in_ad(queue_entry, ldap_conn):
update_entry_ldap(queue_entry, ldap_conn)
else:
create_entry_ldap(queue_entry, ldap_conn)
put_entry_changelog(queue_entry, DIRECTION)
queue_entry_id = queue_entry["id"]
delete_entry_queue(queue_entry_id, OUTBOUND_QUEUE)
def fetch_ldap_data(sync_type, data_type):
"""
Call to get entries for all (Users | Groups) in Active Directory, saves the time of the sync,
and inserts data into RethinkDB.
"""
connect_to_db()
if sync_type == "delta":
last_sync = (
r.table("sync_tracker")
.filter({"source": "ldap-" + data_type})
.coerce_to("array")
.run()
)
last_sync_time = ldap_payload_transformer.to_date_ldap_query(
rethink_timestamp=last_sync[0]["timestamp"]
)
if data_type == "user":
search_filter = LDAP_FILTER_USER_DELTA % last_sync_time
elif data_type == "group":
search_filter = LDAP_FILTER_GROUP_DELTA % last_sync_time
elif sync_type == "initial":
if data_type == "user":
search_filter = LDAP_FILTER_USER
elif data_type == "group":
search_filter = LDAP_FILTER_GROUP
server = Server(LDAP_SERVER, get_info=ALL)
conn = Connection(server, user=LDAP_USER, password=LDAP_PASS)
if not conn.bind():
LOGGER.error(
"Error connecting to LDAP server %s : %s", LDAP_SERVER, conn.result
)
conn.search(
search_base=LDAP_DC,
search_filter=search_filter,
attributes=ldap3.ALL_ATTRIBUTES,
)
insert_to_db(data_dict=conn.entries, data_type=data_type)
sync_source = "ldap-" + data_type
provider_id = LDAP_DC
save_sync_time(provider_id, sync_source, sync_type)
def ldap_sync():
"""Fetches (Users | Groups) from Active Directory and inserts them into RethinkDB."""
if LDAP_DC:
connect_to_db()
db_user_payload = check_last_sync("ldap-user", "initial")
if not db_user_payload:
LOGGER.info(
"No initial AD user sync was found. Starting initial AD user sync now."
)
LOGGER.debug("Inserting AD data...")
LOGGER.debug("Getting Users...")
fetch_ldap_data(sync_type="initial", data_type="user")
LOGGER.debug(
"Initial AD user upload completed. User delta sync will occur in %s seconds.",
str(int(DELTA_SYNC_INTERVAL_SECONDS)),
)
db_group_payload = check_last_sync("ldap-group", "initial")
if not db_group_payload:
LOGGER.info(
"No initial AD group sync was found. Starting initial AD group sync now."
)
LOGGER.debug("Getting Groups with Members...")
fetch_ldap_data(sync_type="initial", data_type="group")
LOGGER.debug(
"Initial AD group upload completed. Group delta sync will occur in %s seconds.",
str(int(DELTA_SYNC_INTERVAL_SECONDS)),
)
if db_user_payload and db_group_payload:
LOGGER.info("The initial sync has already been run.")
# TODO: Recreate threads for delta syncs
else:
LOGGER.debug("LDAP Domain Controller is not provided, skipping LDAP sync.")
def inbound_sync_listener():
"""Initialize a delta inbound sync with Azure Active Directory."""
while True: # pylint: disable=too-many-nested-blocks
provider_id = TENANT_ID
try:
LOGGER.info("Connecting to RethinkDB...")
connect_to_db()
LOGGER.info("Successfully connected to RethinkDB!")
initial_sync_time = check_last_sync("azure-user", "initial")
initial_sync_time = initial_sync_time[0]["timestamp"][:26]
latest_delta_sync_time = get_last_delta_sync(provider_id, "delta")
if latest_delta_sync_time:
latest_delta_sync_time = latest_delta_sync_time[
"timestamp"][:26]
previous_sync_datetime = datetime.strptime(
latest_delta_sync_time, "%Y-%m-%dT%H:%M:%S.%f")
else:
previous_sync_datetime = datetime.strptime(
initial_sync_time, "%Y-%m-%dT%H:%M:%S.%f")
# Create an eventhub client.
client = EventHubClient(ADDRESS,
debug=False,
username=USER,
password=KEY)
try:
LOGGER.info("Opening connection to EventHub...")
# Set prefetch to 1, we only want one event at a time.
receiver = client.add_receiver(CONSUMER_GROUP,
PARTITION,
prefetch=1,
offset=OFFSET)
# Open the connection to the EventHub.
client.run()
# Get one event from EventHub.
batch = receiver.receive(timeout=5000)
while batch:
for event_data in batch:
# Get the event as a json record from the batch of events.
event_json = event_data.body_as_json()
record = event_json["records"][0]
operation_name = record["operationName"]
time = record["time"][:26]
record_timestamp = datetime.strptime(
time, "%Y-%m-%dT%H:%M:%S.%f")
# Only process events logged after the previous initial/delta sync.
# Only grab events concerning User or Group objects.
if (operation_name in VALID_OPERATIONS
and record_timestamp > previous_sync_datetime):
data = {
"initated_by":
record["properties"]["initiatedBy"],
"target_resources":
record["properties"]["targetResources"],
"operation_name":
operation_name,
"resultType":
record["resultType"],
}
LOGGER.info("Operation name: %s", operation_name)
LOGGER.info("Record to Change: %s", record)
record_timestamp_utc = record_timestamp.isoformat()
insert_change_to_db(data, record_timestamp_utc)
sync_source = "azure-" + VALID_OPERATIONS[
operation_name]
provider_id = TENANT_ID
save_sync_time(provider_id, sync_source, "delta",
record_timestamp_utc)
previous_sync_datetime = record_timestamp
batch = receiver.receive(timeout=50)
LOGGER.info("Closing connection to EventHub...")
# Close the connection to the EventHub.
client.stop()
except KeyboardInterrupt:
pass
finally:
client.stop()
except ExpectedError as err:
LOGGER.debug(
("%s Repolling after %s seconds...", err.__str__, DELAY))
time.sleep(DELAY)
except Exception as err:
LOGGER.exception(err)
raise err
