def test_import_ocw2hugo_courses( settings, mocked_celery, mocker, filter_str, chunk_size, limit, call_count, delete_unpublished, ): """ import_ocw2hugo_course_paths should be called correct # times for given chunk size, limit, filter, and # of paths """ setup_s3(settings) mock_import_paths = mocker.patch( "ocw_import.tasks.import_ocw2hugo_course_paths.si") mock_delete_task = mocker.patch( "ocw_import.tasks.delete_unpublished_courses.si") with pytest.raises(mocked_celery.replace_exception_class): import_ocw2hugo_courses.delay( bucket_name=MOCK_BUCKET_NAME, prefix=TEST_OCW2HUGO_PREFIX, chunk_size=chunk_size, filter_str=filter_str, limit=limit, delete_unpublished=delete_unpublished, ) assert mock_import_paths.call_count == call_count assert mock_delete_task.call_count == (1 if delete_unpublished else 0)
def test_import_ocw2hugo_courses_nobucket(mocker): """ import_ocw2hugo_course_paths should be called correct # times for given chunk size and # of paths """ mock_import_paths = mocker.patch( "ocw_import.tasks.import_ocw2hugo_course_paths.si") with pytest.raises(TypeError): import_ocw2hugo_courses.delay( # pylint:disable=no-value-for-parameter bucket_name=None, prefix=TEST_OCW2HUGO_PREFIX, chunk_size=100) assert mock_import_paths.call_count == 0
def test_import_ocw2hugo_courses_delete_unpublished_false( settings, mocker, mocked_celery): """ import_ocw2hugo_courses should not call delete_unpublished when the argument is false """ mock_delete_unpublished_courses = mocker.patch( "ocw_import.tasks.delete_unpublished_courses.si") setup_s3(settings) with pytest.raises(mocked_celery.replace_exception_class): import_ocw2hugo_courses.delay( bucket_name=MOCK_BUCKET_NAME, prefix=TEST_OCW2HUGO_PREFIX, delete_unpublished=False, ) mock_delete_unpublished_courses.assert_not_called()
def test_import_ocw2hugo_courses_delete_unpublished(settings, mocker, mocked_celery): """ import_ocw2hugo_courses should call delete_unpublished when courses have been removed from the ocw-to-hugo directory """ mock_delete_unpublished_courses = mocker.patch( "ocw_import.tasks.delete_unpublished_courses.si") tmpdir = TemporaryDirectory() setup_s3_tmpdir(settings, tmpdir.name) with pytest.raises(mocked_celery.replace_exception_class): import_ocw2hugo_courses.delay(bucket_name=MOCK_BUCKET_NAME, prefix=TEST_OCW2HUGO_PREFIX) mock_delete_unpublished_courses.assert_called_with(paths=ALL_COURSES_PATHS) tmpdir.cleanup() tmpdir = TemporaryDirectory() setup_s3_tmpdir(settings, tmpdir.name, courses=["1-050-engineering-mechanics-i-fall-2007"]) with pytest.raises(mocked_celery.replace_exception_class): import_ocw2hugo_courses.delay(bucket_name=MOCK_BUCKET_NAME, prefix=TEST_OCW2HUGO_PREFIX) mock_delete_unpublished_courses.assert_called_with( paths=SINGLE_COURSE_PATHS) tmpdir.cleanup()
def handle(self, *args, **options): prefix = options["prefix"] if prefix: # make sure it ends with a '/' prefix = prefix.rstrip("/") + "/" bucket_name = options["bucket"] filter_str = options["filter"] limit = options["limit"] delete_unpublished = options["delete_unpublished"] delete_from_git = options["delete_from_git"] if options["list"] is True: course_paths = list( fetch_ocw2hugo_course_paths(bucket_name, prefix=prefix, filter_list=[filter_str])) pydoc.pager("\n".join(course_paths)) return self.stdout.write(f"Importing OCW courses from '{bucket_name}' bucket") start = now_in_utc() task = import_ocw2hugo_courses.delay( bucket_name=bucket_name, prefix=prefix, filter_str=filter_str, limit=limit, delete_unpublished=delete_unpublished, chunk_size=options["chunks"], ) self.stdout.write(f"Starting task {task}...") task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write("OCW course import finished, took {} seconds".format( total_seconds)) if settings.CONTENT_SYNC_BACKEND and not options["skip_sync"]: self.stdout.write( "Syncing all unsynced courses to the designated backend") start = now_in_utc() task = sync_unsynced_websites.delay( create_backends=True, delete=delete_from_git, ) self.stdout.write(f"Starting task {task}...") task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Backend sync finished, took {} seconds".format(total_seconds))