def confirm_changes(options, orgs, org_coursekey_pairs):
"""
Should we apply the changes to the database?
If `--apply`, this just returns True.
If `--dry`, this does a dry run and then returns False.
Otherwise, it does a dry run and then prompts the user.
Arguments:
options (dict[str]): command-line arguments.
orgs (list[dict]): list of org data dictionaries to bulk-add.
org_coursekey_pairs (list[tuple[dict, CourseKey]]):
list of (org data dictionary, course key) links to bulk-add.
Returns: bool
"""
if options.get('apply') and options.get('dry'):
raise CommandError("Only one of 'apply' and 'dry' may be specified")
if options.get('apply'):
return True
organizations_api.bulk_add_organizations(orgs, dry_run=True)
organizations_api.bulk_add_organization_courses(org_coursekey_pairs, dry_run=True)
if options.get('dry'):
return False
answer = ""
while answer.lower() not in {'y', 'yes', 'n', 'no'}:
answer = input('Commit changes shown above to the database [y/n]? ')
return answer.lower().startswith('y')
def test_add_several_organizations(self):
"""
Test that the query_count of bulk_add_organizations does not increase
when given more organizations.
"""
existing_org = api.add_organization(
self.make_organization_data("existing_org"))
api.remove_organization(
api.add_organization(
self.make_organization_data("org_to_reactivate"))["id"])
# 1 query to load list of existing orgs,
# 1 query to filter for only inactive existing orgs,
# 1 query for activate-existing, and 1 query for create-new.
with self.assertNumQueries(4):
api.bulk_add_organizations([
existing_org,
existing_org,
existing_org,
self.make_organization_data("org_to_reactivate"),
self.make_organization_data("new_org_1"),
self.make_organization_data("new_org_2"),
self.make_organization_data("new_org_3"),
self.make_organization_data("new_org_4"),
self.make_organization_data("new_org_5"),
self.make_organization_data("new_org_6"),
self.make_organization_data("new_org_7"),
self.make_organization_data("new_org_8"),
self.make_organization_data("new_org_9"),
self.make_organization_data("new_org_9"), # Redundant.
self.make_organization_data("new_org_9"), # Redundant.
])
assert len(api.get_organizations()) == 11
def handle(self, *args, **options):
"""
Handle the backfill command.
"""
orgslug_coursekey_pairs = find_orgslug_coursekey_pairs()
orgslug_library_pairs = find_orgslug_library_pairs()
orgslugs = (
{orgslug for orgslug, _ in orgslug_coursekey_pairs} |
{orgslug for orgslug, _ in orgslug_library_pairs}
)
# Note: the `organizations.api.bulk_add_*` code will handle:
# * not overwriting existing organizations, and
# * skipping duplicates, based on the short name (case-insensiive),
# so we don't have to worry about those here.
orgs = [
{"short_name": orgslug, "name": orgslug}
# The `sorted` calls aren't strictly necessary, but they'll help make this
# function more deterministic in case something goes wrong.
for orgslug in sorted(orgslugs)
]
org_coursekey_pairs = [
({"short_name": orgslug}, coursekey)
for orgslug, coursekey in sorted(orgslug_coursekey_pairs)
]
if not confirm_changes(options, orgs, org_coursekey_pairs):
print("No changes applied.")
return
print("Applying changes...")
organizations_api.bulk_add_organizations(orgs, dry_run=False)
organizations_api.bulk_add_organization_courses(org_coursekey_pairs, dry_run=False)
print("Changes applied successfully.")
def test_dry_run(self, mock_log_info):
"""
Test that `bulk_add_organizations` does nothing when `dry_run` is
specified (except logging).
"""
api.bulk_add_organizations(
[self.make_organization_data("org_a")],
dry_run=True,
)
assert api.get_organizations() == []
# One for reactivations, one for creations.
assert mock_log_info.call_count == 2
def test_validation_errors(self):
"""
Test the `bulk_add_organizations` raises validation errors on bad input,
and no organizations are created.
"""
with self.assertRaises(exceptions.InvalidOrganizationException):
api.bulk_add_organizations([
self.make_organization_data("valid_org"),
{
"description": "org with no short_name!"
},
])
assert len(api.get_organizations()) == 0
def bulk_add_data(
orgs: List[dict],
org_courseid_pairs: List[Tuple[dict, str]],
dry_run: bool,
activate: bool,
):
"""
Bulk-add the organizations and organization-course linkages.
Print out list of organizations and organization-course linkages,
one per line. We distinguish between records that are added by
being created vs. those that are being added by just reactivating an
existing record.
Arguments:
orgs: org data dictionaries to bulk-add.
should each have a "short_name" and "name" key.
org_courseid_pairs
list of (org data dictionary, course key string) links to bulk-add.
each org data dictionary should have a "short_name" key.
dry_run: Whether or not this run should be "dry" (ie, don't apply changes).
activate: Whether newly-added organizations and organization-course linkages
should be activated, and whether existing-but-inactive
organizations/linkages should be reactivated.
"""
adding_phrase = "Dry-run of bulk-adding" if dry_run else "Bulk-adding"
created_phrase = "Will create" if dry_run else "Created"
reactivated_phrase = "Will reactivate" if dry_run else "Reactivated"
print("------------------------------------------------------")
print(f"{adding_phrase} organizations...")
orgs_created, orgs_reactivated = organizations_api.bulk_add_organizations(
orgs, dry_run=dry_run, activate=activate)
print(f"{created_phrase} {len(orgs_created)} organizations:")
for org_short_name in sorted(orgs_created):
print(f" {org_short_name}")
print(f"{reactivated_phrase} {len(orgs_reactivated)} organizations:")
for org_short_name in sorted(orgs_reactivated):
print(f" {org_short_name}")
print("------------------------------------------------------")
print(f"{adding_phrase} organization-course linkages...")
linkages_created, linkages_reactivated = organizations_api.bulk_add_organization_courses(
org_courseid_pairs, dry_run=dry_run, activate=activate)
print(
f"{created_phrase} {len(linkages_created)} organization-course linkages:"
)
for org_short_name, course_id in sorted(linkages_created):
print(f" {org_short_name},{course_id}")
print(
f"{reactivated_phrase} {len(linkages_reactivated)} organization-course linkages:"
)
for org_short_name, course_id in sorted(linkages_reactivated):
print(f" {org_short_name},{course_id}")
print("------------------------------------------------------")
def test_edge_cases(self, mock_log_info):
"""
Test that bulk_add_organizations handles a few edge cases as expected.
"""
# Add three orgs, and remove all but the first.
# Use capitalized name to confirm case insensitivity when checking
# for existing orgs.
api.add_organization(self.make_organization_data("EXISTING_ORG"))
api.remove_organization(
api.add_organization(
self.make_organization_data("org_to_reactivate"))["id"])
api.remove_organization(
api.add_organization(
self.make_organization_data("org_to_leave_inactive"))["id"])
# 1 query to load list of existing orgs,
# 1 query to filter for only inactive existing orgs,
# 1 query for create, and 1 query for update.
with self.assertNumQueries(4):
api.bulk_add_organizations([
# New organization.
self.make_organization_data("org_X"),
# Modify existing active organization; should be no-op.
{
**self.make_organization_data("existing_org"), "description":
"this name should be ignored"
},
# Deleted organizations are still stored in the DB as "inactive".
# Bulk-adding should reactivate it.
self.make_organization_data("org_to_reactivate"),
# Another new organizaiton.
self.make_organization_data("org_Y"),
# Another org with same short name (case-insensitively)
# as first new organization; should be ignored.
{
**self.make_organization_data("ORG_x"), "name":
"this name should be ignored"
}
])
# There should exist the already-existing org, the org that existed as inactive
# but is not activated, and the two new orgs.
# This should not include `org_to_leave_inactive`.
organizations = api.get_organizations()
assert {organization["short_name"]
for organization in organizations
} == {"EXISTING_ORG", "org_to_reactivate", "org_X", "org_Y"}
# Organization dicts with already-taken short_names shouldn't have modified
# the existing orgs.
assert "this name should be ignored" not in {
organization["name"]
for organization in organizations
}
# Based on logging messages, make sure we dropped the appropriate
# organization dict from the bulk-add batch.
logging_of_drop_from_batch = mock_log_info.call_args_list[0][0]
assert logging_of_drop_from_batch[1]["short_name"] == (
# We dropped this org data:
self.make_organization_data("ORG_x")["short_name"])
assert logging_of_drop_from_batch[2]["short_name"] == (
# in favor of this org data, which came earlier in the batch.
self.make_organization_data("org_X")["short_name"])
# Based on logging messages, make sure the expected breakdown of
# created vs. reactivated vs. not touched
# is true for the organizations passed to `bulk_add_organizations`.
logged_orgs_to_reactivate = mock_log_info.call_args_list[1][0][2]
assert set(logged_orgs_to_reactivate) == {"org_to_reactivate"}
logged_orgs_to_create = mock_log_info.call_args_list[2][0][2]
assert set(logged_orgs_to_create) == {"org_X", "org_Y"}
def test_add_no_organizations(self):
"""
Test that `bulk_add_organizations` is a no-op when given an empty list.
"""
api.bulk_add_organizations([])
assert len(api.get_organizations()) == 0