def test_delete_image_type_asset(self):
""" Tests deletion of image type asset """
image_asset = self.get_sample_asset(self.asset_name, asset_type="image")
thumbnail_image_asset = self.get_sample_asset('delete_test_thumbnail', asset_type="image")
# upload image
response = self.client.post(self.url, {"name": "delete_image_test", "file": image_asset})
self.assertEquals(response.status_code, 200)
uploaded_image_url = json.loads(response.content)['asset']['url']
# upload image thumbnail
response = self.client.post(self.url, {"name": "delete_image_thumb_test", "file": thumbnail_image_asset})
self.assertEquals(response.status_code, 200)
thumbnail_url = json.loads(response.content)['asset']['url']
thumbnail_location = StaticContent.get_location_from_path(thumbnail_url)
image_asset_location = AssetLocation.from_deprecated_string(uploaded_image_url)
content = contentstore().find(image_asset_location)
content.thumbnail_location = thumbnail_location
contentstore().save(content)
with mock.patch('opaque_keys.edx.locator.CourseLocator.make_asset_key') as mock_asset_key:
mock_asset_key.return_value = thumbnail_location
test_url = reverse_course_url(
'assets_handler', self.course.id, kwargs={'asset_key_string': unicode(uploaded_image_url)})
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEquals(resp.status_code, 204)
def _upload_asset(request, course_key):
'''
This method allows for POST uploading of files into the course asset
library, which will be supported by GridFS in MongoDB.
'''
# Does the course actually exist?!? Get anything from it to prove its
# existence
try:
modulestore().get_course(course_key)
except ItemNotFoundError:
# no return it as a Bad Request response
logging.error("Could not find course: %s", course_key)
return HttpResponseBadRequest()
# compute a 'filename' which is similar to the location formatting, we're
# using the 'filename' nomenclature since we're using a FileSystem paradigm
# here. We're just imposing the Location string formatting expectations to
# keep things a bit more consistent
upload_file = request.FILES['file']
filename = upload_file.name
mime_type = upload_file.content_type
content_loc = StaticContent.compute_location(course_key, filename)
chunked = upload_file.multiple_chunks()
sc_partial = partial(StaticContent, content_loc, filename, mime_type)
if chunked:
content = sc_partial(upload_file.chunks())
tempfile_path = upload_file.temporary_file_path()
else:
content = sc_partial(upload_file.read())
tempfile_path = None
# first let's see if a thumbnail can be created
(thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(
content,
tempfile_path=tempfile_path
)
# delete cached thumbnail even if one couldn't be created this time (else
# the old thumbnail will continue to show)
del_cached_content(thumbnail_location)
# now store thumbnail location only if we could create it
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
contentstore().save(content)
del_cached_content(content.location)
# readback the saved content - we need the database timestamp
readback = contentstore().find(content.location)
locked = getattr(content, 'locked', False)
response_payload = {
'asset': _get_asset_json(content.name, readback.last_modified_at, content.location, content.thumbnail_location, locked),
'msg': _('Upload completed')
}
return JsonResponse(response_payload)
def delete_asset(course_key, asset_key):
"""
Deletes asset represented by given 'asset_key' in the course represented by given course_key.
"""
# Make sure the item to delete actually exists.
try:
content = contentstore().find(asset_key)
except NotFoundError:
raise AssetNotFoundException
# ok, save the content into the trashcan
contentstore('trashcan').save(content)
# see if there is a thumbnail as well, if so move that as well
if content.thumbnail_location is not None:
# We are ignoring the value of the thumbnail_location-- we only care whether
# or not a thumbnail has been stored, and we can now easily create the correct path.
thumbnail_location = course_key.make_asset_key('thumbnail', asset_key.name)
try:
thumbnail_content = contentstore().find(thumbnail_location)
contentstore('trashcan').save(thumbnail_content)
# hard delete thumbnail from origin
contentstore().delete(thumbnail_content.get_id())
# remove from any caching
del_cached_content(thumbnail_location)
except Exception: # pylint: disable=broad-except
logging.warning('Could not delete thumbnail: %s', thumbnail_location)
# delete the original
contentstore().delete(content.get_id())
# remove from cache
del_cached_content(content.location)
def upload_image(course_key, upload_file):
filename = upload_file.name
mime_type = upload_file.content_type
content_loc = StaticContent.compute_location(course_key, filename)
chunked = upload_file.multiple_chunks()
sc_partial = partial(StaticContent, content_loc, filename, mime_type)
if chunked:
content = sc_partial(upload_file.chunks())
tempfile_path = upload_file.temporary_file_path()
else:
content = sc_partial(upload_file.read())
tempfile_path = None
# first let's see if a thumbnail can be created
(thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(
content,
tempfile_path=tempfile_path
)
# delete cached thumbnail even if one couldn't be created this time (else
# the old thumbnail will continue to show)
del_cached_content(thumbnail_location)
# now store thumbnail location only if we could create it
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
contentstore().save(content)
del_cached_content(content.location)
return filename
def _upload_file(videoId, lang, location):
if lang == 'en':
filename = 'subs_{0}.srt.sjson'.format(videoId)
else:
filename = '{0}_subs_{1}.srt.sjson'.format(lang, videoId)
path = os.path.join(TEST_ROOT, 'uploads/', filename)
f = open(os.path.abspath(path))
mime_type = "application/json"
content_location = StaticContent.compute_location(
location.org, location.course, filename
)
sc_partial = partial(StaticContent, content_location, filename, mime_type)
content = sc_partial(f.read())
(thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(
content,
tempfile_path=None
)
del_cached_content(thumbnail_location)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
contentstore().save(content)
del_cached_content(content.location)
def handle(self, *args, **options):
try:
# a course key may have unicode chars in it
try:
course_key = text_type(options['course_key'], 'utf8')
# May already be decoded to unicode if coming in through tests, this is ok.
except TypeError:
course_key = text_type(options['course_key'])
course_key = CourseKey.from_string(course_key)
except InvalidKeyError:
raise CommandError(u'Invalid course_key: {}'.format(options['course_key']))
if not modulestore().get_course(course_key):
raise CommandError(u'Course not found: {}'.format(options['course_key']))
print(u'Preparing to delete course %s from module store....' % options['course_key'])
if query_yes_no(u'Are you sure you want to delete course {}?'.format(course_key), default='no'):
if query_yes_no(u'Are you sure? This action cannot be undone!', default='no'):
delete_course(course_key, ModuleStoreEnum.UserID.mgmt_command, options['keep_instructors'])
if options['remove_assets']:
contentstore().delete_all_course_assets(course_key)
print(u'Deleted assets for course'.format(course_key))
print(u'Deleted course {}'.format(course_key))
def drop_mongo_collections(store_name="default"):
"""
If using a Mongo-backed modulestore & contentstore, drop the collections.
"""
# This will return the mongo-backed modulestore
# even if we're using a mixed modulestore
store = editable_modulestore(store_name)
if hasattr(store, "collection"):
connection = store.collection.database.connection
store.collection.drop()
connection.close()
elif hasattr(store, "close_all_connections"):
store.close_all_connections()
elif hasattr(store, "db"):
connection = store.db.connection
connection.drop_database(store.db.name)
connection.close()
if contentstore().fs_files:
db = contentstore().fs_files.database
db.connection.drop_database(db)
db.connection.close()
location_mapper = loc_mapper()
if location_mapper.db:
location_mapper.location_map.drop()
location_mapper.db.connection.close()
def drop_mongo_collections(modulestore_type=MONGO_MODULESTORE_TYPE):
"""
If using a Mongo-backed modulestore & contentstore, drop the collections.
"""
store = modulestore()
if hasattr(store, '_get_modulestore_by_type'):
store = store._get_modulestore_by_type(modulestore_type) # pylint: disable=W0212
if hasattr(store, 'collection'):
connection = store.collection.database.connection
store.collection.drop()
connection.close()
elif hasattr(store, 'close_all_connections'):
store.close_all_connections()
elif hasattr(store, 'db'):
connection = store.db.connection
connection.drop_database(store.db.name)
connection.close()
if contentstore().fs_files:
db = contentstore().fs_files.database
db.connection.drop_database(db)
db.connection.close()
location_mapper = loc_mapper()
if location_mapper.db:
location_mapper.location_map.drop()
location_mapper.db.connection.close()
def _update_asset(request, course_key, asset_key):
"""
restful CRUD operations for a course asset.
Currently only DELETE, POST, and PUT methods are implemented.
asset_path_encoding: the odd /c4x/org/course/category/name repr of the asset (used by Backbone as the id)
"""
if request.method == 'DELETE':
try:
delete_asset(course_key, asset_key)
return JsonResponse()
except AssetNotFoundException:
return JsonResponse(status=404)
elif request.method in ('PUT', 'POST'):
if 'file' in request.FILES:
return _upload_asset(request, course_key)
else:
# Update existing asset
try:
modified_asset = json.loads(request.body)
except ValueError:
return HttpResponseBadRequest()
contentstore().set_attr(asset_key, 'locked', modified_asset['locked'])
# Delete the asset from the cache so we check the lock status the next time it is requested.
del_cached_content(asset_key)
return JsonResponse(modified_asset, status=201)
def update_course_run_asset(course_key, upload_file):
course_exists_response = _get_error_if_course_does_not_exist(course_key)
if course_exists_response is not None:
return course_exists_response
file_metadata = _get_file_metadata_as_dictionary(upload_file)
is_file_too_large = _check_file_size_is_too_large(file_metadata)
if is_file_too_large:
error_message = _get_file_too_large_error_message(file_metadata['filename'])
raise AssetSizeTooLargeException(error_message)
content, temporary_file_path = _get_file_content_and_path(file_metadata, course_key)
(thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(content,
tempfile_path=temporary_file_path)
# delete cached thumbnail even if one couldn't be created this time (else the old thumbnail will continue to show)
del_cached_content(thumbnail_location)
if _check_thumbnail_uploaded(thumbnail_content):
content.thumbnail_location = thumbnail_location
contentstore().save(content)
del_cached_content(content.location)
return content
def setUpClass(cls):
cls.courses = {}
super(CanonicalContentTest, cls).setUpClass()
names_and_prefixes = [(ModuleStoreEnum.Type.split, 'split'), (ModuleStoreEnum.Type.mongo, 'old')]
for store, prefix in names_and_prefixes:
with cls.store.default_store(store):
cls.courses[prefix] = CourseFactory.create(org='a', course='b', run=prefix)
# Create an unlocked image.
unlock_content = cls.create_image(prefix, (32, 32), 'blue', u'{}_ünlöck.png')
# Create a locked image.
lock_content = cls.create_image(prefix, (32, 32), 'green', '{}_lock.png', locked=True)
# Create a thumbnail of the images.
contentstore().generate_thumbnail(unlock_content, dimensions=(16, 16))
contentstore().generate_thumbnail(lock_content, dimensions=(16, 16))
# Create an unlocked image in a subdirectory.
cls.create_image(prefix, (1, 1), 'red', u'special/{}_ünlöck.png')
# Create a locked image in a subdirectory.
cls.create_image(prefix, (1, 1), 'yellow', 'special/{}_lock.png', locked=True)
# Create an unlocked image with funky characters in the name.
cls.create_image(prefix, (1, 1), 'black', u'weird {}_ünlöck.png')
cls.create_image(prefix, (1, 1), 'black', u'special/weird {}_ünlöck.png')
# Create an HTML file to test extension exclusion, and create a control file.
cls.create_arbitrary_content(prefix, '{}_not_excluded.htm')
cls.create_arbitrary_content(prefix, '{}_excluded.html')
cls.create_arbitrary_content(prefix, 'special/{}_not_excluded.htm')
cls.create_arbitrary_content(prefix, 'special/{}_excluded.html')
def test_can_delete_certificate_with_signatories(self):
"""
Delete certificate
"""
self._add_course_certificates(count=2, signatory_count=1)
certificates = self.course.certificates["certificates"]
org_logo_url = certificates[1]["org_logo_path"]
image_asset_location = AssetLocation.from_deprecated_string(org_logo_url)
content = contentstore().find(image_asset_location)
self.assertIsNotNone(content)
response = self.client.delete(
self._url(cid=1),
content_type="application/json",
HTTP_ACCEPT="application/json",
HTTP_X_REQUESTED_WITH="XMLHttpRequest",
)
self.assertEqual(response.status_code, 204)
self.assert_event_emitted(
"edx.certificate.configuration.deleted", course_id=unicode(self.course.id), configuration_id="1"
)
self.reload_course()
# Verify that certificates are properly updated in the course.
certificates = self.course.certificates["certificates"]
self.assertEqual(len(certificates), 1)
# make sure certificate org logo is deleted too
self.assertRaises(NotFoundError, contentstore().find, image_asset_location)
self.assertEqual(certificates[0].get("name"), "Name 0")
self.assertEqual(certificates[0].get("description"), "Description 0")
def test_can_delete_signatory(self):
"""
Delete an existing certificate signatory
"""
self._add_course_certificates(count=2, signatory_count=3)
certificates = self.course.certificates["certificates"]
signatory = certificates[1].get("signatories")[1]
image_asset_location = AssetLocation.from_deprecated_string(signatory["signature_image_path"])
content = contentstore().find(image_asset_location)
self.assertIsNotNone(content)
test_url = "{}/signatories/1".format(self._url(cid=1))
response = self.client.delete(
test_url,
content_type="application/json",
HTTP_ACCEPT="application/json",
HTTP_X_REQUESTED_WITH="XMLHttpRequest",
)
self.assertEqual(response.status_code, 204)
self.reload_course()
# Verify that certificates are properly updated in the course.
certificates = self.course.certificates["certificates"]
self.assertEqual(len(certificates[1].get("signatories")), 2)
# make sure signatory signature image is deleted too
self.assertRaises(NotFoundError, contentstore().find, image_asset_location)
def clear_subs_content(self):
"""Remove, if subtitles content exists."""
try:
content = contentstore().find(self.content_location)
contentstore().delete(content.get_id())
except NotFoundError:
pass
def test_images_upload(self):
# http://www.django-rest-framework.org/api-guide/parsers/#fileuploadparser
course_run = CourseFactory()
expected_filename = 'course_image.png'
content_key = StaticContent.compute_location(course_run.id, expected_filename)
assert course_run.course_image != expected_filename
try:
contentstore().find(content_key)
self.fail('No image should be associated with a new course run.')
except NotFoundError:
pass
url = reverse('api:v1:course_run-images', kwargs={'pk': str(course_run.id)})
# PNG. Single black pixel
content = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS' \
b'\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00IEND\xaeB`\x82'
# We are intentionally passing the incorrect JPEG extension here
upload = SimpleUploadedFile('card_image.jpg', content, content_type='image/png')
response = self.client.post(url, {'card_image': upload}, format='multipart')
assert response.status_code == 200
course_run = modulestore().get_course(course_run.id)
assert course_run.course_image == expected_filename
expected = {'card_image': RequestFactory().get('').build_absolute_uri(course_image_url(course_run))}
assert response.data == expected
# There should now be an image stored
contentstore().find(content_key)
def test_fail_downloading_subs(self):
# Disabled 11/14/13
# This test is flakey because it performs an HTTP request on an external service
# Re-enable when `requests.get` is patched using `mock.patch`
raise SkipTest
bad_youtube_subs = {
0.5: 'BAD_YOUTUBE_ID1',
1.0: 'BAD_YOUTUBE_ID2',
2.0: 'BAD_YOUTUBE_ID3'
}
self.clear_subs_content(bad_youtube_subs)
with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
transcripts_utils.download_youtube_subs(bad_youtube_subs, self.course)
# Check assets status after importing subtitles.
for subs_id in bad_youtube_subs.values():
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(
self.org, self.number, filename
)
with self.assertRaises(NotFoundError):
contentstore().find(content_location)
self.clear_subs_content(bad_youtube_subs)
def test_different_resolutions(self, src_dimensions):
"""
Test various resolutions of images to make thumbnails of.
Note that our test sizes are small=(200, 100) and large=(400, 200).
1. Images should won't be blown up if it's too small, so a (100, 50)
resolution image will remain (100, 50).
2. However, images *will* be converted using our format and quality
settings (JPEG, 75% -- the PIL default). This is because images with
relatively small dimensions not compressed properly.
3. Image thumbnail naming will maintain the naming convention of the
target resolution, even if the image was not actually scaled to that
size (i.e. it was already smaller). This is mostly because it's
simpler to be consistent, but it also lets us more easily tell which
configuration a thumbnail was created under.
"""
# Create a source image...
image = Image.new("RGB", src_dimensions, "blue")
image_buff = StringIO()
image.save(image_buff, format="PNG")
image_buff.seek(0)
image_name = "src_course_image.png"
course = CourseFactory.create(course_image=image_name)
# Save the image to the contentstore...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff)
contentstore().save(course_image_content)
# Now generate the CourseOverview...
config = CourseOverviewImageConfig.current()
course_overview = CourseOverview.get_from_id(course.id)
image_urls = course_overview.image_urls
for image_url, target in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
# Naming convention for thumbnail
self.assertTrue(image_url.endswith('src_course_image-png-{}x{}.jpg'.format(*target)))
# Actual thumbnail data
src_x, src_y = src_dimensions
target_x, target_y = target
image_x, image_y = image.size
# I'm basically going to assume the image library knows how to do
# the right thing in terms of handling aspect ratio. We're just
# going to make sure that small images aren't blown up, and that
# we never exceed our target sizes
self.assertLessEqual(image_x, target_x)
self.assertLessEqual(image_y, target_y)
if src_x < target_x and src_y < target_y:
self.assertEqual(src_x, image_x)
self.assertEqual(src_y, image_y)
def test_happy_path(self, modulestore_type, create_after_overview):
"""
What happens when everything works like we expect it to.
If `create_after_overview` is True, we will temporarily disable
thumbnail creation so that the initial CourseOverview is created without
an image_set, and the CourseOverviewImageSet is created afterwards. If
`create_after_overview` is False, we'll create the CourseOverviewImageSet
at the same time as the CourseOverview.
"""
# Create a real (oversized) image...
image = Image.new("RGB", (800, 400), "blue")
image_buff = StringIO()
image.save(image_buff, format="JPEG")
image_buff.seek(0)
image_name = "big_course_image.jpeg"
with self.store.default_store(modulestore_type):
course = CourseFactory.create(
default_store=modulestore_type, course_image=image_name
)
# Save a real image here...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/jpeg', image_buff)
contentstore().save(course_image_content)
# If create_after_overview is True, disable thumbnail generation so
# that the CourseOverview object is created and saved without an
# image_set at first (it will be lazily created later).
if create_after_overview:
self.set_config(enabled=False)
# Now generate the CourseOverview...
course_overview = CourseOverview.get_from_id(course.id)
# If create_after_overview is True, no image_set exists yet. Verify
# that, then switch config back over to True and it should lazily
# create the image_set on the next get_from_id() call.
if create_after_overview:
self.assertFalse(hasattr(course_overview, 'image_set'))
self.set_config(enabled=True)
course_overview = CourseOverview.get_from_id(course.id)
self.assertTrue(hasattr(course_overview, 'image_set'))
image_urls = course_overview.image_urls
config = CourseOverviewImageConfig.current()
# Make sure the thumbnail names come out as expected...
self.assertTrue(image_urls['raw'].endswith('big_course_image.jpeg'))
self.assertTrue(image_urls['small'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.small)))
self.assertTrue(image_urls['large'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.large)))
# Now make sure our thumbnails are of the sizes we expect...
for image_url, expected_size in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
self.assertEqual(image.size, expected_size)
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']}))
raise
except Exception as exc:
LOGGER.exception('There was an error exporting %s', course_key, exc_info=True)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
def _create_fake_images(self, asset_keys):
"""
Creates fake image files for a list of asset_keys.
"""
for asset_key_string in asset_keys:
asset_key = AssetKey.from_string(asset_key_string)
content = StaticContent(asset_key, "Fake asset", "image/png", "data")
contentstore().save(content)
def clear_subs_content(self):
"""Remove, if subtitles content exists."""
for content_location in [self.content_location, self.content_copied_location]:
try:
content = contentstore().find(content_location)
contentstore().delete(content.location)
except NotFoundError:
pass
def test_delete_asset_with_invalid_thumbnail(self):
""" Tests the sad path :( """
test_url = reverse_course_url(
'assets_handler', self.course.id, kwargs={'asset_key_string': unicode(self.uploaded_url)})
self.content.thumbnail_location = StaticContent.get_location_from_path('/c4x/edX/toy/asset/invalid')
contentstore().save(self.content)
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEquals(resp.status_code, 204)
def upload_to_local(self):
content_loc = StaticContent.compute_location(self.course_key, self.filename)
mime_type = 'application/json'
# Note: cribbed from common/lib/xmodule/xmodule/video_module/transcripts_utils.py save_subs_to_store()
filedata = json.dumps(self.subs, indent=2)
content = StaticContent(content_loc, self.filename, mime_type, filedata)
contentstore().save(content)
del_cached_content(content_loc)
def delete_asset(course_key, asset_key):
content = _check_existence_and_get_asset_content(asset_key)
_save_content_to_trash(content)
_delete_thumbnail(content.thumbnail_location, course_key, asset_key)
contentstore().delete(content.get_id())
del_cached_content(content.location)
def _upload_file(subs_file, location, filename):
mime_type = subs_file.content_type
content_location = StaticContent.compute_location(
location.course_key, filename
)
content = StaticContent(content_location, filename, mime_type, subs_file.read())
contentstore().save(content)
del_cached_content(content.location)
def test_save_subs_to_store(self):
with self.assertRaises(NotFoundError):
contentstore().find(self.content_location)
result_location = transcripts_utils.save_subs_to_store(self.subs, self.subs_id, self.course)
self.assertTrue(contentstore().find(self.content_location))
self.assertEqual(result_location, self.content_location)
def clear_courses():
# Flush and initialize the module store
# Note that if your test module gets in some weird state
# (though it shouldn't), do this manually
# from the bash shell to drop it:
# $ mongo test_xmodule --eval "db.dropDatabase()"
editable_modulestore().collection.drop()
contentstore().fs_files.drop()
def upload_file(filename, location):
path = os.path.join(TEST_ROOT, "uploads/", filename)
f = open(os.path.abspath(path))
mime_type = "application/json"
content_location = StaticContent.compute_location(location.course_key, filename)
content = StaticContent(content_location, filename, mime_type, f.read())
contentstore().save(content)
del_cached_content(content.location)
def clear_courses():
# Flush and initialize the module store
# Note that if your test module gets in some weird state
# (though it shouldn't), do this manually
# from the bash shell to drop it:
# $ mongo test_xmodule --eval "db.dropDatabase()"
store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
store.collection.drop()
contentstore().fs_files.drop()
def create_export_tarball(course_module, course_key, context):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting %s', course_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
raise
except Exception as exc:
log.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'unit': None,
'raw_err_msg': str(exc)})
raise
finally:
shutil.rmtree(root_dir / name)
return export_file
def generate_sjson_for_all_speeds(item, user_filename, result_subs_dict, lang):
"""
Generates sjson from srt for given lang.
`item` is module object.
"""
try:
srt_transcript = contentstore().find(
Transcript.asset_location(item.location, user_filename))
except NotFoundError as ex:
raise TranscriptException(
"{}: Can't find uploaded transcripts: {}".format(
ex.message, user_filename))
if not lang:
lang = item.transcript_language
generate_subs_from_source(result_subs_dict,
os.path.splitext(user_filename)[1][1:],
srt_transcript.data.decode('utf8'), item, lang)
def test_asset_and_course_deletion(self):
course_run = CourseFactory()
self.assertIsNotNone(modulestore().get_course(course_run.id))
store = contentstore()
asset_key = course_run.id.make_asset_key('asset', 'test.txt')
content = StaticContent(asset_key, 'test.txt', 'plain/text',
'test data')
store.save(content)
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 1
with mock.patch(self.YESNO_PATCH_LOCATION) as patched_yes_no:
patched_yes_no.return_value = True
call_command('delete_course', str(course_run.id))
assert modulestore().get_course(course_run.id) is None
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 1
def copy_or_rename_transcript(new_name,
old_name,
item,
delete_old=False,
user=None):
"""
Renames `old_name` transcript file in storage to `new_name`.
If `old_name` is not found in storage, raises `NotFoundError`.
If `delete_old` is True, removes `old_name` files from storage.
"""
filename = u'subs_{0}.srt.sjson'.format(old_name)
content_location = StaticContent.compute_location(item.location.course_key,
filename)
transcripts = contentstore().find(content_location).data.decode('utf-8')
save_subs_to_store(json.loads(transcripts), new_name, item)
item.sub = new_name
item.save_with_metadata(user)
if delete_old:
remove_subs_from_store(old_name, item)
def assertAssetsEqual(self, asset_son, course1_id, course2_id):
"""Verifies the asset of the given key has the same attributes in both given courses."""
content_store = contentstore()
category = asset_son.block_type if hasattr(
asset_son, 'block_type') else asset_son['category']
filename = asset_son.block_id if hasattr(
asset_son, 'block_id') else asset_son['name']
course1_asset_attrs = content_store.get_attrs(
course1_id.make_asset_key(category, filename))
course2_asset_attrs = content_store.get_attrs(
course2_id.make_asset_key(category, filename))
self.assertEqual(len(course1_asset_attrs), len(course2_asset_attrs))
for key, value in course1_asset_attrs.iteritems():
if key in [
'_id', 'filename', 'uploadDate', 'content_son',
'thumbnail_location'
]:
pass
else:
self.assertEqual(value, course2_asset_attrs[key])
def test_asset_import_nostatic(self):
'''
This test validates that an image asset is NOT imported when do_import_static=False
'''
content_store = contentstore()
module_store = modulestore('direct')
import_from_xml(module_store,
'common/test/data/', ['toy'],
static_content_store=content_store,
do_import_static=False,
verbose=True)
course_location = CourseDescriptor.id_to_location('edX/toy/2012_Fall')
module_store.get_item(course_location)
# make sure we have NO assets in our contentstore
all_assets = content_store.get_all_content_for_course(course_location)
print "len(all_assets)=%d" % len(all_assets)
self.assertEqual(len(all_assets), 0)
def download_transcripts(request):
"""
Passes to user requested transcripts file.
Raises Http404 if unsuccessful.
"""
locator = request.GET.get('locator')
if not locator:
log.debug('GET data without "locator" property.')
raise Http404
try:
item = _get_item(request, request.GET)
except (InvalidKeyError, ItemNotFoundError):
log.debug("Can't find item by locator.")
raise Http404
subs_id = request.GET.get('subs_id')
if not subs_id:
log.debug('GET data without "subs_id" property.')
raise Http404
if item.category != 'video':
log.debug('transcripts are supported only for video" modules.')
raise Http404
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(item.location.course_key, filename)
try:
sjson_transcripts = contentstore().find(content_location)
log.debug("Downloading subs for %s id", subs_id)
str_subs = generate_srt_from_sjson(json.loads(sjson_transcripts.data), speed=1.0)
if not str_subs:
log.debug('generate_srt_from_sjson produces no subtitles')
raise Http404
response = HttpResponse(str_subs, content_type='application/x-subrip')
response['Content-Disposition'] = 'attachment; filename="{0}.srt"'.format(subs_id)
return response
except NotFoundError:
log.debug("Can't find content in storage for %s subs", subs_id)
raise Http404
def setUp(self):
"""
Create user and login.
"""
settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
self.client = Client()
self.contentstore = contentstore()
# A locked asset
self.loc_locked = Location('c4x', 'edX', 'toy', 'asset', 'sample_static.txt')
self.url_locked = StaticContent.get_url_path_from_location(self.loc_locked)
# An unlocked asset
self.loc_unlocked = Location('c4x', 'edX', 'toy', 'asset', 'another_static.txt')
self.url_unlocked = StaticContent.get_url_path_from_location(self.loc_unlocked)
import_from_xml(modulestore('direct'), 'common/test/data/', ['toy'],
static_content_store=self.contentstore, verbose=True)
self.contentstore.set_attr(self.loc_locked, 'locked', True)
# Create user
self.usr = '******'
self.pwd = 'foo'
email = '*****@*****.**'
self.user = User.objects.create_user(self.usr, email, self.pwd)
self.user.is_active = True
self.user.save()
# Create staff user
self.staff_usr = '******'
self.staff_pwd = 'foo'
staff_email = '*****@*****.**'
self.staff_user = User.objects.create_user(self.staff_usr, staff_email,
self.staff_pwd)
self.staff_user.is_active = True
self.staff_user.is_staff = True
self.staff_user.save()
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError(
"clone requires two arguments: <source-location> <dest-location>"
)
source_location_str = args[0]
dest_location_str = args[1]
ms = modulestore('direct')
cs = contentstore()
print "Cloning course {0} to {1}".format(source_location_str,
dest_location_str)
source_location = CourseDescriptor.id_to_location(source_location_str)
dest_location = CourseDescriptor.id_to_location(dest_location_str)
if clone_course(ms, cs, source_location, dest_location):
print "copying User permissions..."
_copy_course_group(source_location, dest_location)
def modulestore():
"""
Returns the Mixed modulestore
"""
global _MIXED_MODULESTORE # pylint: disable=global-statement
if _MIXED_MODULESTORE is None:
_MIXED_MODULESTORE = create_modulestore_instance(
settings.MODULESTORE['default']['ENGINE'], contentstore(),
settings.MODULESTORE['default'].get('DOC_STORE_CONFIG', {}),
settings.MODULESTORE['default'].get('OPTIONS', {}))
if settings.FEATURES.get('CUSTOM_COURSES_EDX'):
# TODO: This import prevents a circular import issue, but is
# symptomatic of a lib having a dependency on code in lms. This
# should be updated to have a setting that enumerates modulestore
# wrappers and then uses that setting to wrap the modulestore in
# appropriate wrappers depending on enabled features.
from lms.djangoapps.ccx.modulestore import CCXModulestoreWrapper
_MIXED_MODULESTORE = CCXModulestoreWrapper(_MIXED_MODULESTORE)
return _MIXED_MODULESTORE
def read_from_contentstore(course_key, path):
"""
Loads a file directly from the course's content store.
"""
contents = None
try:
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from opaque_keys.edx.locator import CourseLocator
except ImportError:
# We're not running under edx-platform, so ignore.
pass
else:
if isinstance(course_key, six.text_type):
course_key = CourseLocator.from_string(course_key)
loc = StaticContent.compute_location(course_key, path)
asset = contentstore().find(loc)
contents = asset.data
return contents
def load_test_import_course(self):
'''
Load the standard course used to test imports
(for do_import_static=False behavior).
'''
content_store = contentstore()
module_store = modulestore('direct')
import_from_xml(
module_store,
'common/test/data/',
['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True,
)
course_id = SlashSeparatedCourseKey('edX', 'test_import_course',
'2012_Fall')
course = module_store.get_course(course_id)
self.assertIsNotNone(course)
return module_store, content_store, course
def load_test_import_course(self):
'''
Load the standard course used to test imports
(for do_import_static=False behavior).
'''
content_store = contentstore()
module_store = modulestore('direct')
import_from_xml(
module_store,
'common/test/data/',
['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True,
)
course_location = CourseDescriptor.id_to_location(
'edX/test_import_course/2012_Fall')
course = module_store.get_item(course_location)
self.assertIsNotNone(course)
return module_store, content_store, course, course_location
def handle(self, *args, **options):
"""
Execute the command
"""
content_store = contentstore()
success = False
log.info(u"-" * 80)
log.info(u"Cleaning up assets for all courses")
try:
# Remove all redundant Mac OS metadata files
assets_deleted = content_store.remove_redundant_content_for_courses()
success = True
except Exception as err:
log.info(u"=" * 30 + u"> failed to cleanup")
log.info(u"Error:")
log.info(err)
if success:
log.info(u"=" * 80)
log.info(u"Total number of assets deleted: {0}".format(assets_deleted))
def test_export_course_with_unknown_metadata(self):
module_store = modulestore('direct')
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['full'])
location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
root_dir = path(mkdtemp_clean())
course = module_store.get_item(location)
metadata = own_metadata(course)
# add a bool piece of unknown metadata so we can verify we don't throw an exception
metadata['new_metadata'] = True
module_store.update_metadata(location, metadata)
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(module_store, content_store, location, root_dir, 'test_export')
def test_asset_import_nostatic(self):
'''
This test validates that an image asset is NOT imported when do_import_static=False
'''
content_store = contentstore()
module_store = modulestore()
import_from_xml(module_store,
self.user.id,
'common/test/data/', ['toy'],
static_content_store=content_store,
do_import_static=False,
verbose=True)
course = module_store.get_course(
SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
# make sure we have NO assets in our contentstore
all_assets, count = content_store.get_all_content_for_course(course.id)
self.assertEqual(len(all_assets), 0)
self.assertEqual(count, 0)
def handle(self, *args, **options):
"Execute the command"
if len(args) == 0:
raise CommandError(
"import requires at least one argument: <data directory> [--nostatic] [<course dir>...]"
)
data_dir = args[0]
do_import_static = not (options.get('nostatic', False))
if len(args) > 1:
course_dirs = args[1:]
else:
course_dirs = None
self.stdout.write(
"Importing. Data_dir={data}, course_dirs={courses}\n".format(
data=data_dir, courses=course_dirs, dis=do_import_static))
try:
mstore = modulestore('direct')
except KeyError:
self.stdout.write('Unable to load direct modulestore, trying '
'default\n')
mstore = modulestore('default')
_, course_items = import_from_xml(
mstore,
data_dir,
course_dirs,
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=do_import_static,
create_new_course=True,
)
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
self.stdout.write(
'Seeding forum roles for course {0}\n'.format(course_id))
seed_permissions_roles(course_id)
def test_success_downloading_chinese_transcripts(self):
# Disabled 11/14/13
# This test is flakey because it performs an HTTP request on an external service
# Re-enable when `requests.get` is patched using `mock.patch`
raise SkipTest
good_youtube_sub = 'j_jEn79vS3g' # Chinese, utf-8
self.clear_sub_content(good_youtube_sub)
# Check transcripts_utils.GetTranscriptsFromYouTubeException not thrown
transcripts_utils.download_youtube_subs(good_youtube_sub, self.course,
settings)
# Check assets status after importing subtitles.
for subs_id in good_youtube_subs.values():
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(
self.course.id, filename)
self.assertTrue(contentstore().find(content_location))
self.clear_sub_content(good_youtube_sub)
def get_transcript(self, subs_id):
'''
Returns transcript without timecodes.
Args:
`subs_id`: str, subtitles id
Raises:
- NotFoundError if cannot find transcript file in storage.
- ValueError if transcript file is incorrect JSON.
- KeyError if transcript file has incorrect format.
'''
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(
self.location.org, self.location.course, filename
)
data = contentstore().find(content_location).data
text = json.loads(data)['text']
return HTMLParser().unescape("\n".join(text))
def test_asset_import_nostatic(self):
'''
This test validates that an image asset is NOT imported when do_import_static=False
'''
content_store = contentstore()
module_store = modulestore()
import_course_from_xml(module_store,
self.user.id,
TEST_DATA_DIR, ['toy'],
static_content_store=content_store,
do_import_static=False,
create_if_not_present=True,
verbose=True)
course = module_store.get_course(
module_store.make_course_key('edX', 'toy', '2012_Fall'))
# make sure we have NO assets in our contentstore
all_assets, count = content_store.get_all_content_for_course(course.id)
self.assertEqual(len(all_assets), 0)
self.assertEqual(count, 0)
def handle(self, *args, **options):
"""
Execute the command
"""
content_store = contentstore()
success = False
log.info("-" * 80)
log.info("Cleaning up assets for all courses")
try:
# Remove all redundant Mac OS metadata files
assets_deleted = content_store.remove_redundant_content_for_courses(
)
success = True
except Exception as err: # lint-amnesty, pylint: disable=broad-except
log.info("=" * 30 + "> failed to cleanup") # lint-amnesty, pylint: disable=logging-not-lazy
log.info("Error:")
log.info(err)
if success:
log.info("=" * 80)
log.info(f"Total number of assets deleted: {assets_deleted}")
def setUpClass(cls):
super(ContentStoreToyCourseTest, cls).setUpClass()
cls.contentstore = contentstore()
cls.modulestore = modulestore()
cls.course_key = cls.modulestore.make_course_key('edX', 'toy', '2012_Fall')
import_course_from_xml(
cls.modulestore, 1, TEST_DATA_DIR, ['toy'],
static_content_store=cls.contentstore, verbose=True
)
# A locked asset
cls.locked_asset = cls.course_key.make_asset_key('asset', 'sample_static.txt')
cls.url_locked = unicode(cls.locked_asset)
cls.contentstore.set_attr(cls.locked_asset, 'locked', True)
# An unlocked asset
cls.unlocked_asset = cls.course_key.make_asset_key('asset', 'another_static.txt')
cls.url_unlocked = unicode(cls.unlocked_asset)
cls.length_unlocked = cls.contentstore.get_attr(cls.unlocked_asset, 'length')
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError(
"export requires two arguments: <course id> <output path>")
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(
args[0])
output_path = args[1]
print("Exporting course id = {0} to {1}".format(
course_key, output_path))
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore('direct'), contentstore(), course_key,
root_dir, course_dir, modulestore())
def handle(self, *args, **options):
"Execute the command"
if len(args) == 0:
raise CommandError(
"import requires at least one argument: <data directory> [--nostatic] [<course dir>...]"
)
data_dir = args[0]
do_import_static = not (options.get('nostatic', False))
if len(args) > 1:
course_dirs = args[1:]
else:
course_dirs = None
print("Importing. Data_dir={data}, course_dirs={courses}".format(
data=data_dir, courses=course_dirs, dis=do_import_static))
import_from_xml(modulestore('direct'),
data_dir,
course_dirs,
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=do_import_static)
def find(asset_key, throw_on_not_found=True, as_stream=False):
"""
Finds a course asset either in the assetstore -or- in the deprecated contentstore.
"""
store = modulestore()
content_md = None
asset_type = asset_key.asset_type
if asset_type == AssetThumbnailMetadata.ASSET_TYPE:
content_md = store.find_asset_thumbnail_metadata(asset_key)
elif asset_type == AssetMetadata.ASSET_TYPE:
content_md = store.find_asset_metadata(asset_key)
else:
raise UnknownAssetType()
# If found, raise an exception.
if content_md:
# For now, no asset metadata should be found in the modulestore.
raise AssetMetadataFoundTemporary()
else:
# If not found, load the asset via the contentstore.
return contentstore().find(asset_key, throw_on_not_found,
as_stream)
def delete_course_and_groups(course_id, commit=False):
"""
This deletes the courseware associated with a course_id as well as cleaning update_item
the various user table stuff (groups, permissions, etc.)
"""
module_store = modulestore('direct')
content_store = contentstore()
module_store.ignore_write_events_on_courses.add(course_id)
if delete_course(module_store, content_store, course_id, commit):
print 'removing User permissions from course....'
# in the django layer, we need to remove all the user permissions groups associated with this course
if commit:
try:
staff_role = CourseStaffRole(course_id)
staff_role.remove_users(*staff_role.users_with_role())
instructor_role = CourseInstructorRole(course_id)
instructor_role.remove_users(*instructor_role.users_with_role())
except Exception as err:
log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))
def handle(self, *args, **options):
data_dir = options['data_directory']
source_dirs = options['course_dirs']
if not source_dirs:
source_dirs = None
do_import_static = not options.get('nostatic', False)
# If the static content is not skipped, the python lib should be imported regardless
# of the 'nopythonlib' flag.
do_import_python_lib = do_import_static or not options.get('nopythonlib', False)
python_lib_filename = options.get('python_lib_filename')
output = (
u"Importing...\n"
u" data_dir={data}, source_dirs={courses}\n"
u" Importing static content? {import_static}\n"
u" Importing python lib? {import_python_lib}"
).format(
data=data_dir,
courses=source_dirs,
import_static=do_import_static,
import_python_lib=do_import_python_lib
)
self.stdout.write(output)
mstore = modulestore()
course_items = import_course_from_xml(
mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, source_dirs, load_error_modules=False,
static_content_store=contentstore(), verbose=True,
do_import_static=do_import_static, do_import_python_lib=do_import_python_lib,
create_if_not_present=True,
python_lib_filename=python_lib_filename,
)
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
self.stdout.write(u'Seeding forum roles for course {0}\n'.format(course_id))
seed_permissions_roles(course_id)
def student_view(self, context=None):
"""
The student view of the MarkdownXBlock.
"""
if self.filename:
# These can only be imported when the XBlock is running on the LMS
# or CMS. Do it at runtime so that the workbench is usable for
# regular XML content.
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
from opaque_keys import InvalidKeyError
try:
course_id = self.xmodule_runtime.course_id
loc = StaticContent.compute_location(course_id, self.filename)
asset = contentstore().find(loc)
content = asset.data
except (NotFoundError, InvalidKeyError):
pass
else:
content = self.content
html_content = ""
if content:
html_content = markdown2.markdown(content, extras=self.extras)
# Render the HTML template
context = {'content': html_content}
html = loader.render_template('templates/main.html', context)
frag = Fragment(html)
if "fenced-code-blocks" in self.extras:
frag.add_css_url(
self.runtime.local_resource_url(self,
'public/css/pygments.css'))
return frag
def handle(self, *args, **options):
"Execute the command"
if len(args) == 0:
raise CommandError(
"import requires at least one argument: <data directory> [--nostatic] [<course dir>...]"
)
data_dir = args[0]
do_import_static = not (options.get('nostatic', False))
if len(args) > 1:
source_dirs = args[1:]
else:
source_dirs = None
self.stdout.write(
"Importing. Data_dir={data}, source_dirs={courses}\n".format(
data=data_dir,
courses=source_dirs,
))
mstore = modulestore()
course_items = import_course_from_xml(
mstore,
ModuleStoreEnum.UserID.mgmt_command,
data_dir,
source_dirs,
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=do_import_static,
create_if_not_present=True,
)
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
self.stdout.write(
'Seeding forum roles for course {0}\n'.format(course_id))
seed_permissions_roles(course_id)
def test_branching(self):
"""
Exercise branching code of import
"""
repo_dir = self.GIT_REPO_DIR
# Test successful import from command
if not os.path.isdir(repo_dir):
os.mkdir(repo_dir)
self.addCleanup(shutil.rmtree, repo_dir)
# Checkout non existent branch
with self.assertRaisesRegexp(GitImportError,
GitImportError.REMOTE_BRANCH_MISSING):
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite',
'asdfasdfasdf')
# Checkout new branch
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite',
self.TEST_BRANCH)
def_ms = modulestore()
# Validate that it is different than master
self.assertIsNotNone(def_ms.get_course(self.TEST_BRANCH_COURSE))
# Attempt to check out the same branch again to validate branch choosing
# works
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite',
self.TEST_BRANCH)
# Delete to test branching back to master
delete_course(def_ms, contentstore(), self.TEST_BRANCH_COURSE, True)
self.assertIsNone(def_ms.get_course(self.TEST_BRANCH_COURSE))
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite',
'master')
self.assertIsNone(def_ms.get_course(self.TEST_BRANCH_COURSE))
self.assertIsNotNone(
def_ms.get_course(
SlashSeparatedCourseKey.from_deprecated_string(
self.TEST_COURSE)))
def test_downloading_subs_using_transcript_name(self, mock_get):
"""
Download transcript using transcript name in url
"""
good_youtube_sub = 'good_id_2'
self.clear_sub_content(good_youtube_sub)
transcripts_utils.download_youtube_subs(good_youtube_sub, self.course,
settings)
mock_get.assert_any_call('http://video.google.com/timedtext',
params={
'lang': 'en',
'v': 'good_id_2',
'name': 'Custom'
})
# Check asset status after import of transcript.
filename = 'subs_{0}.srt.sjson'.format(good_youtube_sub)
content_location = StaticContent.compute_location(
self.course.id, filename)
self.assertTrue(contentstore().find(content_location))
self.clear_sub_content(good_youtube_sub)
