def test_import_ocw2hugo_courses(
settings,
mocked_celery,
mocker,
filter_str,
chunk_size,
limit,
call_count,
delete_unpublished,
):
"""
import_ocw2hugo_course_paths should be called correct # times for given chunk size, limit, filter, and # of paths
"""
setup_s3(settings)
mock_import_paths = mocker.patch(
"ocw_import.tasks.import_ocw2hugo_course_paths.si")
mock_delete_task = mocker.patch(
"ocw_import.tasks.delete_unpublished_courses.si")
with pytest.raises(mocked_celery.replace_exception_class):
import_ocw2hugo_courses.delay(
bucket_name=MOCK_BUCKET_NAME,
prefix=TEST_OCW2HUGO_PREFIX,
chunk_size=chunk_size,
filter_str=filter_str,
limit=limit,
delete_unpublished=delete_unpublished,
)
assert mock_import_paths.call_count == call_count
assert mock_delete_task.call_count == (1 if delete_unpublished else 0)
def test_import_ocw2hugo_courses_nobucket(mocker):
""" import_ocw2hugo_course_paths should be called correct # times for given chunk size and # of paths """
mock_import_paths = mocker.patch(
"ocw_import.tasks.import_ocw2hugo_course_paths.si")
with pytest.raises(TypeError):
import_ocw2hugo_courses.delay( # pylint:disable=no-value-for-parameter
bucket_name=None,
prefix=TEST_OCW2HUGO_PREFIX,
chunk_size=100)
assert mock_import_paths.call_count == 0
def test_import_ocw2hugo_courses_delete_unpublished_false(
settings, mocker, mocked_celery):
""" import_ocw2hugo_courses should not call delete_unpublished when the argument is false """
mock_delete_unpublished_courses = mocker.patch(
"ocw_import.tasks.delete_unpublished_courses.si")
setup_s3(settings)
with pytest.raises(mocked_celery.replace_exception_class):
import_ocw2hugo_courses.delay(
bucket_name=MOCK_BUCKET_NAME,
prefix=TEST_OCW2HUGO_PREFIX,
delete_unpublished=False,
)
mock_delete_unpublished_courses.assert_not_called()
def test_import_ocw2hugo_courses_delete_unpublished(settings, mocker,
mocked_celery):
""" import_ocw2hugo_courses should call delete_unpublished when courses have been removed from the ocw-to-hugo directory """
mock_delete_unpublished_courses = mocker.patch(
"ocw_import.tasks.delete_unpublished_courses.si")
tmpdir = TemporaryDirectory()
setup_s3_tmpdir(settings, tmpdir.name)
with pytest.raises(mocked_celery.replace_exception_class):
import_ocw2hugo_courses.delay(bucket_name=MOCK_BUCKET_NAME,
prefix=TEST_OCW2HUGO_PREFIX)
mock_delete_unpublished_courses.assert_called_with(paths=ALL_COURSES_PATHS)
tmpdir.cleanup()
tmpdir = TemporaryDirectory()
setup_s3_tmpdir(settings,
tmpdir.name,
courses=["1-050-engineering-mechanics-i-fall-2007"])
with pytest.raises(mocked_celery.replace_exception_class):
import_ocw2hugo_courses.delay(bucket_name=MOCK_BUCKET_NAME,
prefix=TEST_OCW2HUGO_PREFIX)
mock_delete_unpublished_courses.assert_called_with(
paths=SINGLE_COURSE_PATHS)
tmpdir.cleanup()
def handle(self, *args, **options):
prefix = options["prefix"]
if prefix:
# make sure it ends with a '/'
prefix = prefix.rstrip("/") + "/"
bucket_name = options["bucket"]
filter_str = options["filter"]
limit = options["limit"]
delete_unpublished = options["delete_unpublished"]
delete_from_git = options["delete_from_git"]
if options["list"] is True:
course_paths = list(
fetch_ocw2hugo_course_paths(bucket_name,
prefix=prefix,
filter_list=[filter_str]))
pydoc.pager("\n".join(course_paths))
return
self.stdout.write(f"Importing OCW courses from '{bucket_name}' bucket")
start = now_in_utc()
task = import_ocw2hugo_courses.delay(
bucket_name=bucket_name,
prefix=prefix,
filter_str=filter_str,
limit=limit,
delete_unpublished=delete_unpublished,
chunk_size=options["chunks"],
)
self.stdout.write(f"Starting task {task}...")
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write("OCW course import finished, took {} seconds".format(
total_seconds))
if settings.CONTENT_SYNC_BACKEND and not options["skip_sync"]:
self.stdout.write(
"Syncing all unsynced courses to the designated backend")
start = now_in_utc()
task = sync_unsynced_websites.delay(
create_backends=True,
delete=delete_from_git,
)
self.stdout.write(f"Starting task {task}...")
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Backend sync finished, took {} seconds".format(total_seconds))