def insert_updated_entries(data_dict, when_changed, data_type): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" insertion_counter = 0 conn = connect_to_db() for entry in data_dict: if entry.whenChanged.value > when_changed: if data_type == "user": standardized_entry = inbound_user_filter(entry, "ldap") else: standardized_entry = inbound_group_filter(entry, "ldap") entry_modified_timestamp = entry.whenChanged.value.strftime( "%Y-%m-%dT%H:%M:%S.%f+00:00") inbound_entry = { "data": standardized_entry, "data_type": data_type, "sync_type": "delta", "timestamp": entry_modified_timestamp, "provider_id": LDAP_DC, } LOGGER.debug( "Inserting LDAP %s into inbound queue: %s", data_type, standardized_entry["remote_id"], ) r.table("inbound_queue").insert(inbound_entry).run(conn) sync_source = "ldap-" + data_type provider_id = LDAP_DC save_sync_time(provider_id, sync_source, "delta", entry_modified_timestamp) insertion_counter += 1 conn.close() LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def test_data_is_kept_when_null(): """Test that a user list stays null when it is None.""" result = inbound_user_filter({ "id": "123-456-abs3", "manager": None }, "azure") assert result["manager_id"] is None
def insert_to_db(entry, data_type): """Insert user or group individually to RethinkDB from dict of data and begins delta sync timer.""" if data_type == "user": standard_entry = inbound_user_filter(entry, "ldap") elif data_type == "group": standard_entry = inbound_group_filter(entry, "ldap") else: LOGGER.warning("unsupported data type: %s", data_type) return inbound_entry = { "data": standard_entry, "data_type": data_type, "sync_type": "initial", "timestamp": r.now(), "provider_id": LDAP_DC, } conn = connect_to_db() LOGGER.debug( "Inserting LDAP %s into inbound queue: %s", data_type, standard_entry["remote_id"], ) r.table("inbound_queue").insert(inbound_entry).run(conn) conn.close()
def test_data_kept_with_empty_lst(): """Test that a list stays a list when there is no value in it.""" result = inbound_user_filter({ "id": "123-456-abs3", "manager": [] }, "azure") assert result["manager_id"] == []
def test_inbound_user_filter(): """Test the inbound user filter for azure transforms and returns a user dict.""" result = inbound_user_filter({"id": 1234}, "azure") assert isinstance(result, dict) is True assert result["user_id"] == 1234 assert "id" not in result assert result["job_title"] is None
def insert_to_db(data_dict, when_changed): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" insertion_counter = 0 for entry in data_dict: if entry.whenChanged.value > when_changed: entry_data = json.loads(entry.entry_to_json())["attributes"] if "person" in entry.objectClass.value: data_type = "user" standardized_entry = inbound_user_filter(entry_data, "ldap") else: data_type = "group" standardized_entry = inbound_group_filter(entry_data, "ldap") entry_modified_timestamp = entry.whenChanged.value.strftime( "%Y-%m-%dT%H:%M:%S.%f+00:00") inbound_entry = { "data": standardized_entry, "data_type": data_type, "sync_type": "delta", "timestamp": entry_modified_timestamp, "provider_id": LDAP_DC, } r.table("inbound_queue").insert(inbound_entry).run() sync_source = "ldap-" + data_type provider_id = LDAP_DC save_sync_time(provider_id, sync_source, "delta", entry_modified_timestamp) insertion_counter += 1 LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def insert_to_db(data_dict, data_type): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" for entry in data_dict: entry_to_insert = {} entry_json = json.loads(entry.entry_to_json()) entry_attributes = entry_json["attributes"] for attribute in entry_attributes: if len(entry_attributes[attribute]) > 1: entry_to_insert[attribute] = entry_attributes[attribute] else: entry_to_insert[attribute] = entry_attributes[attribute][0] if data_type == "user": standardized_entry = inbound_user_filter(entry_to_insert, "ldap") elif data_type == "group": standardized_entry = inbound_group_filter(entry_to_insert, "ldap") inbound_entry = { "data": standardized_entry, "data_type": data_type, "sync_type": "initial", "timestamp": datetime.now().replace(tzinfo=timezone.utc).isoformat(), "provider_id": LDAP_DC, } r.table("inbound_queue").insert(inbound_entry).run() LOGGER.info( "Inserted %s %s records into inbound_queue.", str(len(data_dict)), data_type )
def test_user_data_type_correct(): """Test that a list stays a list when a single value is in it.""" result = inbound_user_filter( { "id": "123-456-abs3", "manager": ["123-456-abs3"] }, "azure") assert result["manager_id"] == ["123-456-abs3"]
def insert_user_to_db(users_dict): """Insert users individually to rethinkdb from dict of users. This will also look up a user's manager.""" for user in users_dict["value"]: manager = fetch_user_manager(user["id"]) if manager: user["manager"] = manager["id"] else: user["manager"] = "" standardized_user = inbound_user_filter(user, "azure") inbound_entry = { "data": standardized_user, "data_type": "user", "sync_type": "initial", "timestamp": dt.now().isoformat(), "provider_id": TENANT_ID, } r.table("queue_inbound").insert(inbound_entry).run()
def insert_to_db(entry, data_type): """Insert user or group individually to RethinkDB from dict of data and begins delta sync timer.""" if data_type == "user": standard_entry = inbound_user_filter(entry, "ldap") elif data_type == "group": standard_entry = inbound_group_filter(entry, "ldap") else: LOGGER.warning("unsupported data type: %s", data_type) return inbound_entry = { "data": standard_entry, "data_type": data_type, "sync_type": "initial", "timestamp": r.now(), "provider_id": LDAP_DC, } add_transaction(inbound_entry) r.table("inbound_queue").insert(inbound_entry).run()
def insert_to_db(data_dict, when_changed, data_type): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" insertion_counter = 0 conn = connect_to_db() for entry in data_dict: entry_to_insert = {} entry_json = json.loads(entry.entry_to_json()) entry_attributes = entry_json["attributes"] for attribute in entry_attributes: if len(entry_attributes[attribute]) > 1: entry_to_insert[attribute] = entry_attributes[attribute] else: entry_to_insert[attribute] = entry_attributes[attribute][0] if entry.whenChanged.value > when_changed: if data_type == "user": standardized_entry = inbound_user_filter( entry_to_insert, "ldap") else: standardized_entry = inbound_group_filter( entry_to_insert, "ldap") entry_modified_timestamp = entry.whenChanged.value.strftime( "%Y-%m-%dT%H:%M:%S.%f+00:00") inbound_entry = { "data": standardized_entry, "data_type": data_type, "sync_type": "delta", "timestamp": entry_modified_timestamp, "provider_id": LDAP_DC, } add_transaction(inbound_entry) r.table("inbound_queue").insert(inbound_entry).run(conn) sync_source = "ldap-" + data_type provider_id = LDAP_DC save_sync_time(provider_id, sync_source, "delta", entry_modified_timestamp) insertion_counter += 1 conn.close() LOGGER.info("Inserted %s records into inbound_queue.", insertion_counter)
def insert_to_db(data_dict, data_type): """Insert (Users | Groups) individually to RethinkDB from dict of data and begins delta sync timer.""" for entry in data_dict: entry_data = json.loads(entry.entry_to_json())["attributes"] if data_type == "user": standardized_entry = inbound_filters.inbound_user_filter(entry_data, "ldap") elif data_type == "group": standardized_entry = inbound_filters.inbound_group_filter( entry_data, "ldap" ) inbound_entry = { "data": standardized_entry, "data_type": data_type, "timestamp": datetime.now().replace(tzinfo=timezone.utc).isoformat(), "provider_id": LDAP_DC, } r.table("queue_inbound").insert(inbound_entry).run() LOGGER.info( "Inserted %s %s records into inbound_queue.", str(len(data_dict)), data_type ) Timer( DELTA_SYNC_INTERVAL_SECONDS, fetch_ldap_data, args=("delta", data_type) ).start()
def test_inbound_user_filter_bad_provider(): """Test the inbound user filter with bad provider throws error""" with pytest.raises(TypeError): inbound_user_filter({"id": 1234}, "potato")
def test_inbound_user_filter(): """Test the inbound user filter for azure transforms and returns a user dict.""" result = inbound_user_filter({"id": "123-456-abs3"}, "azure") assert isinstance(result, dict) is True assert result["remote_id"] == "123-456-abs3" assert "id" not in result