def test_activate_channel(self):
previous_tree = self.channel.previous_tree
tree(parent=previous_tree)
garbage_node = get_deleted_chefs_root()
# Previous tree shouldn't be in garbage tree until activate_channel is called
self.assertFalse(garbage_node.get_descendants().filter(
pk=previous_tree.pk).exists())
activate_channel(self.channel, self.user)
garbage_node.refresh_from_db()
previous_tree.refresh_from_db()
self.channel.refresh_from_db()
# We can't use MPTT methods on the deleted chefs tree because we are not running the sort code
# for performance reasons, so just do a parent test instead.
self.assertTrue(previous_tree.parent == garbage_node)
# New previous tree should not be in garbage tree
self.assertFalse(self.channel.previous_tree.parent)
self.assertNotEqual(garbage_node.tree_id,
self.channel.previous_tree.tree_id)
child_pk = previous_tree.children.first().pk
clean_up_deleted_chefs()
self.assertFalse(
cc.ContentNode.objects.filter(parent=garbage_node).exists())
self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def test_old_chef_tree(self):
# make an actual tree for deletion tests
tree(parent=self.channel.chef_tree)
chef_tree = self.channel.chef_tree
self.assertTrue(chef_tree.get_descendant_count() > 0)
garbage_node = get_deleted_chefs_root()
self.assertNotEqual(chef_tree, self.channel.staging_tree)
# Chef tree shouldn't be in garbage tree until create_channel is called
self.assertFalse(
garbage_node.get_descendants().filter(pk=chef_tree.pk).exists())
create_channel(self.channel.__dict__, self.user)
garbage_node.refresh_from_db()
chef_tree.refresh_from_db()
self.channel.refresh_from_db()
# We can't use MPTT methods to test the deleted chefs tree because we are not running the sort code
# for performance reasons, so just do a parent test instead.
self.assertEquals(chef_tree.parent.pk, garbage_node.pk)
# New staging tree should not be in garbage tree
self.assertFalse(self.channel.chef_tree.parent)
self.assertNotEqual(garbage_node.tree_id,
self.channel.chef_tree.tree_id)
child_pk = chef_tree.children.first().pk
clean_up_deleted_chefs()
self.assertFalse(
cc.ContentNode.objects.filter(parent=garbage_node).exists())
self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def test_old_chef_tree(self):
# make an actual tree for deletion tests
tree(parent=self.channel.chef_tree)
chef_tree = self.channel.chef_tree
self.assertTrue(chef_tree.get_descendant_count() > 0)
garbage_node = get_deleted_chefs_root()
self.assertNotEqual(chef_tree, self.channel.staging_tree)
# Chef tree shouldn't be in garbage tree until create_channel is called
self.assertFalse(garbage_node.get_descendants().filter(pk=chef_tree.pk).exists())
create_channel(self.channel.__dict__, self.user)
garbage_node.refresh_from_db()
chef_tree.refresh_from_db()
self.channel.refresh_from_db()
# We can't use MPTT methods to test the deleted chefs tree because we are not running the sort code
# for performance reasons, so just do a parent test instead.
self.assertEquals(chef_tree.parent.pk, garbage_node.pk)
# New staging tree should not be in garbage tree
self.assertFalse(self.channel.chef_tree.parent)
self.assertNotEqual(garbage_node.tree_id, self.channel.chef_tree.tree_id)
child_pk = chef_tree.children.first().pk
clean_up_deleted_chefs()
self.assertFalse(cc.ContentNode.objects.filter(parent=garbage_node).exists())
self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def test_old_staging_tree(self):
staging_tree = self.channel.staging_tree
garbage_node = get_deleted_chefs_root()
tree(parent=staging_tree)
self.assertTrue(staging_tree.get_descendant_count() > 0)
# Staging tree shouldn't be in garbage tree until api_commit_channel is called
self.assertFalse(garbage_node.get_descendants().filter(pk=staging_tree.pk).exists())
request = self.create_post_request(reverse_lazy('api_finish_channel'), data=json.dumps(
{'channel_id': self.channel.pk}), content_type='application/json')
response = api_commit_channel(request)
self.assertEqual(response.status_code, 200)
garbage_node.refresh_from_db()
staging_tree.refresh_from_db()
self.channel.refresh_from_db()
# We can't use MPTT methods on the deleted chefs tree because we are not running the sort code
# for performance reasons, so just do a parent test instead.
self.assertEqual(staging_tree.parent, garbage_node)
# New staging tree should not be in garbage tree
self.assertFalse(self.channel.main_tree.parent)
self.assertNotEqual(garbage_node.tree_id, self.channel.main_tree.tree_id)
child_pk = staging_tree.children.first().pk
clean_up_deleted_chefs()
self.assertFalse(cc.ContentNode.objects.filter(parent=garbage_node).exists())
self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def activate_channel(channel, user):
user.check_channel_space(channel)
if channel.previous_tree and channel.previous_tree != channel.main_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with models.ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
channel.previous_tree.parent = garbage_node
channel.previous_tree.title = "Previous tree for channel {}".format(
channel.pk)
channel.previous_tree.save()
channel.previous_tree = channel.main_tree
channel.main_tree = channel.staging_tree
channel.staging_tree = None
channel.save()
user.staged_files.all().delete()
user.set_space_used()
change = generate_update_event(
channel.id,
CHANNEL,
{
"root_id": channel.main_tree.id,
"staging_root_id": None
},
)
return change
def test_activate_channel(self):
previous_tree = self.channel.previous_tree
tree(parent=previous_tree)
garbage_node = get_deleted_chefs_root()
# Previous tree shouldn't be in garbage tree until activate_channel is called
self.assertFalse(garbage_node.get_descendants().filter(pk=previous_tree.pk).exists())
activate_channel(self.channel, self.user)
garbage_node.refresh_from_db()
previous_tree.refresh_from_db()
self.channel.refresh_from_db()
# We can't use MPTT methods on the deleted chefs tree because we are not running the sort code
# for performance reasons, so just do a parent test instead.
self.assertTrue(previous_tree.parent == garbage_node)
# New previous tree should not be in garbage tree
self.assertFalse(self.channel.previous_tree.parent)
self.assertNotEqual(garbage_node.tree_id, self.channel.previous_tree.tree_id)
child_pk = previous_tree.children.first().pk
clean_up_deleted_chefs()
self.assertFalse(cc.ContentNode.objects.filter(parent=garbage_node).exists())
self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def create_channel(channel_data, user):
""" Set up channel """
# Set up initial channel
channel, isNew = Channel.objects.get_or_create(id=channel_data['id'])
# Add user as editor if channel is new or channel has no editors
# Otherwise, check if user is an editor
if isNew or channel.editors.count() == 0:
channel.editors.add(user)
elif user not in channel.editors.all():
raise SuspiciousOperation(
"User is not authorized to edit this channel")
channel.name = channel_data['name']
channel.description = channel_data['description']
channel.thumbnail = channel_data['thumbnail']
channel.deleted = False
channel.source_id = channel_data.get('source_id')
channel.source_domain = channel_data.get('source_domain')
channel.source_url = channel_data.get(
'source_domain') if isNew else channel.source_url
channel.ricecooker_version = channel_data.get('ricecooker_version')
channel.language_id = channel_data.get('language')
# older versions of ricecooker won't be sending this field.
if 'tagline' in channel_data:
channel.tagline = channel_data['tagline']
old_chef_tree = channel.chef_tree
is_published = channel.main_tree is not None and channel.main_tree.published
# Set up initial staging tree
channel.chef_tree = ContentNode.objects.create(
title=channel.name,
kind_id=content_kinds.TOPIC,
sort_order=get_next_sort_order(),
published=is_published,
content_id=channel.id,
node_id=channel.id,
source_id=channel.source_id,
source_domain=channel.source_domain,
extra_fields={'ricecooker_version': channel.ricecooker_version},
)
channel.chef_tree.save()
channel.save()
# Delete chef tree if it already exists
if old_chef_tree and old_chef_tree != channel.staging_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
old_chef_tree.parent = garbage_node
old_chef_tree.title = "Old chef tree for channel {}".format(
channel.pk)
old_chef_tree.save()
return channel # Return new channel
def api_commit_channel(request):
"""
Commit the channel chef_tree to staging tree to the main tree.
This view backs the endpoint `/api/internal/finish_channel` called by ricecooker.
"""
data = json.loads(request.body)
try:
channel_id = data['channel_id']
request.user.can_edit(channel_id)
obj = Channel.objects.get(pk=channel_id)
# Need to rebuild MPTT tree pointers since we used `disable_mptt_updates`
ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id)
# set original_channel_id and source_channel_id to self since chef tree
obj.chef_tree.get_descendants(include_self=True).update(
original_channel_id=channel_id, source_channel_id=channel_id)
# replace staging_tree with chef_tree
old_staging = obj.staging_tree
obj.staging_tree = obj.chef_tree
obj.chef_tree = None
obj.save()
# Mark old staging tree for garbage collection
if old_staging and old_staging != obj.main_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
old_staging.parent = garbage_node
old_staging.title = "Old staging tree for channel {}".format(
obj.pk)
old_staging.save()
# If ricecooker --stage flag used, we're done (skip ACTIVATE step), else
# we ACTIVATE the channel, i.e., set the main tree from the staged tree
if not data.get('stage'):
try:
activate_channel(obj, request.user)
except PermissionDenied as e:
return Response(str(e), status=e.status_code)
return Response({
"success": True,
"new_channel": obj.pk,
})
except (Channel.DoesNotExist, PermissionDenied):
return HttpResponseNotFound(
"No channel matching: {}".format(channel_id))
except KeyError as e:
return HttpResponseBadRequest("Required attribute missing: {}".format(
e.message))
except Exception as e:
handle_server_error(request)
return HttpResponseServerError(content=str(e), reason=str(e))
def create_channel(channel_data, user):
""" Set up channel """
# Set up initial channel
channel, isNew = Channel.objects.get_or_create(id=channel_data['id'])
# Add user as editor if channel is new or channel has no editors
# Otherwise, check if user is an editor
if isNew or channel.editors.count() == 0:
channel.editors.add(user)
elif user not in channel.editors.all():
raise SuspiciousOperation("User is not authorized to edit this channel")
channel.name = channel_data['name']
channel.description = channel_data['description']
channel.thumbnail = channel_data['thumbnail']
channel.deleted = False
channel.source_id = channel_data.get('source_id')
channel.source_domain = channel_data.get('source_domain')
channel.ricecooker_version = channel_data.get('ricecooker_version')
channel.language_id = channel_data.get('language')
old_chef_tree = channel.chef_tree
is_published = channel.main_tree is not None and channel.main_tree.published
# Set up initial staging tree
channel.chef_tree = ContentNode.objects.create(
title=channel.name,
kind_id=content_kinds.TOPIC,
sort_order=get_next_sort_order(),
published=is_published,
content_id=channel.id,
node_id=channel.id,
source_id=channel.source_id,
source_domain=channel.source_domain,
extra_fields=json.dumps({'ricecooker_version': channel.ricecooker_version}),
)
channel.chef_tree.save()
channel.save()
# Delete chef tree if it already exists
if old_chef_tree and old_chef_tree != channel.staging_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
old_chef_tree.parent = garbage_node
old_chef_tree.title = "Old chef tree for channel {}".format(channel.pk)
old_chef_tree.save()
return channel # Return new channel
def api_commit_channel(request):
""" Commit the channel staging tree to the main tree """
data = json.loads(request.body)
try:
channel_id = data['channel_id']
obj = Channel.objects.get(pk=channel_id)
# rebuild MPTT tree for this channel (since we set "disable_mptt_updates", and bulk_create doesn't trigger rebuild signals anyway)
ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id)
obj.chef_tree.get_descendants(include_self=True).update(
original_channel_id=channel_id, source_channel_id=channel_id)
old_staging = obj.staging_tree
obj.staging_tree = obj.chef_tree
obj.chef_tree = None
obj.save()
# Delete staging tree if it already exists
if old_staging and old_staging != obj.main_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
old_staging.parent = garbage_node
old_staging.title = "Old staging tree for channel {}".format(
obj.pk)
old_staging.save()
if not data.get(
'stage'
): # If user says to stage rather than submit, skip changing trees at this step
try:
activate_channel(obj, request.user)
except PermissionDenied as e:
return HttpResponseForbidden(str(e))
return HttpResponse(
json.dumps({
"success": True,
"new_channel": obj.pk,
}))
except KeyError as e:
return HttpResponseBadRequest("Required attribute missing: {}".format(
e.message))
except Exception as e:
handle_server_error(request)
return HttpResponseServerError(content=str(e), reason=str(e))
def activate_channel(channel, user):
user.check_channel_space(channel)
if channel.previous_tree and channel.previous_tree != channel.main_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with models.ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
channel.previous_tree.parent = garbage_node
channel.previous_tree.title = "Previous tree for channel {}".format(channel.pk)
channel.previous_tree.save()
channel.previous_tree = channel.main_tree
channel.main_tree = channel.staging_tree
channel.staging_tree = None
channel.save()
user.staged_files.all().delete()
def api_commit_channel(request):
""" Commit the channel staging tree to the main tree """
data = json.loads(request.body)
try:
channel_id = data['channel_id']
obj = Channel.objects.get(pk=channel_id)
# rebuild MPTT tree for this channel (since we set "disable_mptt_updates", and bulk_create doesn't trigger rebuild signals anyway)
ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id)
obj.chef_tree.get_descendants(include_self=True).update(original_channel_id=channel_id,
source_channel_id=channel_id)
old_staging = obj.staging_tree
obj.staging_tree = obj.chef_tree
obj.chef_tree = None
obj.save()
# Delete staging tree if it already exists
if old_staging and old_staging != obj.main_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
old_staging.parent = garbage_node
old_staging.title = "Old staging tree for channel {}".format(obj.pk)
old_staging.save()
if not data.get('stage'): # If user says to stage rather than submit, skip changing trees at this step
try:
activate_channel(obj, request.user)
except PermissionDenied as e:
return HttpResponseForbidden(str(e))
return HttpResponse(json.dumps({
"success": True,
"new_channel": obj.pk,
}))
except KeyError as e:
return HttpResponseBadRequest("Required attribute missing: {}".format(e.message))
except Exception as e:
handle_server_error(request)
return HttpResponseServerError(content=str(e), reason=str(e))
def test_old_staging_tree(self):
staging_tree = self.channel.staging_tree
garbage_node = get_deleted_chefs_root()
tree(parent=staging_tree)
self.assertTrue(staging_tree.get_descendant_count() > 0)
# Staging tree shouldn't be in garbage tree until api_commit_channel is called
self.assertFalse(
garbage_node.get_descendants().filter(pk=staging_tree.pk).exists())
request = self.create_post_request(
reverse_lazy("api_finish_channel"),
data=json.dumps({"channel_id": self.channel.pk}),
content_type="application/json",
)
response = api_commit_channel(request)
self.assertEqual(response.status_code, 200)
garbage_node.refresh_from_db()
staging_tree.refresh_from_db()
self.channel.refresh_from_db()
# We can't use MPTT methods on the deleted chefs tree because we are not running the sort code
# for performance reasons, so just do a parent test instead.
self.assertEqual(staging_tree.parent, garbage_node)
# New staging tree should not be in garbage tree
self.assertFalse(self.channel.main_tree.parent)
self.assertNotEqual(garbage_node.tree_id,
self.channel.main_tree.tree_id)
child_pk = staging_tree.children.first().pk
clean_up_deleted_chefs()
self.assertFalse(
cc.ContentNode.objects.filter(parent=garbage_node).exists())
self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def handle(self, *args, **options):
try:
# Set up variables for restoration process
print("\n\n********** STARTING CHANNEL RESTORATION **********")
start = datetime.datetime.now()
source_id = options['source_id']
target_id = options.get('target') or source_id
# Test connection to database
print("Connecting to database for channel {}...".format(source_id))
tempf = tempfile.NamedTemporaryFile(suffix=".sqlite3",
delete=False)
conn = None
try:
if options.get('download_url'):
response = requests.get(
'{}/content/databases/{}.sqlite3'.format(
options['download_url'], source_id))
for chunk in response:
tempf.write(chunk)
else:
filepath = "/".join(
[settings.DB_ROOT, "{}.sqlite3".format(source_id)])
# Check if database exists
if not default_storage.exists(filepath):
raise IOError("The object requested does not exist.")
with default_storage.open(filepath) as fobj:
shutil.copyfileobj(fobj, tempf)
tempf.close()
conn = sqlite3.connect(tempf.name)
cursor = conn.cursor()
# Start by creating channel
print("Creating channel...")
channel, root_pk = create_channel(conn, target_id)
if options.get('editor'):
channel.editors.add(
models.User.objects.get(email=options['editor']))
channel.save()
# Create root node
root = models.ContentNode.objects.create(
sort_order=models.get_next_sort_order(),
node_id=root_pk,
title=channel.name,
kind_id=content_kinds.TOPIC,
original_channel_id=target_id,
source_channel_id=target_id,
)
# Create nodes mapping to channel
print(" Creating nodes...")
with transaction.atomic():
create_nodes(cursor,
target_id,
root,
download_url=options.get('download_url'))
# TODO: Handle prerequisites
# Delete the previous tree if it exists
old_previous = channel.previous_tree
if old_previous:
old_previous.parent = get_deleted_chefs_root()
old_previous.title = "Old previous tree for channel {}".format(
channel.pk)
old_previous.save()
# Save tree to target tree
channel.previous_tree = channel.main_tree
channel.main_tree = root
channel.save()
finally:
conn and conn.close()
tempf.close()
os.unlink(tempf.name)
# Print stats
print("\n\nChannel has been restored (time: {ms})\n".format(
ms=datetime.datetime.now() - start))
print("\n\n********** RESTORATION COMPLETE **********\n\n")
except EarlyExit as e:
logging.warning(
"Exited early due to {message}.".format(message=e.message))
self.stdout.write(
"You can find your database in {path}".format(path=e.db_path))
def api_commit_channel(request):
"""
Commit the channel chef_tree to staging tree to the main tree.
This view backs the endpoint `/api/internal/finish_channel` called by ricecooker.
"""
data = json.loads(request.body)
try:
channel_id = data['channel_id']
request.user.can_edit(channel_id)
obj = Channel.objects.get(pk=channel_id)
# Need to rebuild MPTT tree pointers since we used `disable_mptt_updates`
ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id)
# set original_channel_id and source_channel_id to self since chef tree
obj.chef_tree.get_descendants(include_self=True).update(original_channel_id=channel_id,
source_channel_id=channel_id)
# replace staging_tree with chef_tree
old_staging = obj.staging_tree
obj.staging_tree = obj.chef_tree
obj.chef_tree = None
obj.save()
# Prepare change event indicating a new staging_tree is available
event = generate_update_event(channel_id, CHANNEL, {
"root_id": obj.main_tree.id,
"staging_root_id": obj.staging_tree.id,
})
# Mark old staging tree for garbage collection
if old_staging and old_staging != obj.main_tree:
# IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server
with ContentNode.objects.disable_mptt_updates():
garbage_node = get_deleted_chefs_root()
old_staging.parent = garbage_node
old_staging.title = "Old staging tree for channel {}".format(obj.pk)
old_staging.save()
# Send event (new staging tree or new main tree) to all channel editors
for editor in obj.editors.all():
add_event_for_user(editor.id, event)
_, task = create_async_task(
"get-node-diff",
request.user,
updated_id=obj.staging_tree.id,
original_id=obj.main_tree.id,
)
# Send response back to the content integration script
return Response({
"success": True,
"new_channel": obj.pk,
"diff_task_id": task.pk,
})
except (Channel.DoesNotExist, PermissionDenied):
return HttpResponseNotFound("No channel matching: {}".format(channel_id))
except KeyError:
return HttpResponseBadRequest("Required attribute missing from data: {}".format(data))
except Exception as e:
handle_server_error(request)
return HttpResponseServerError(content=str(e), reason=str(e))
def import_channel(source_id,
target_id=None,
download_url=None,
editor=None,
logger=None):
"""
Import a channel from another Studio instance. This can be used to
copy online Studio channels into local machines for development,
testing, faster editing, or other purposes.
:param source_id: The UUID of the channel to import from the source Studio instance.
:param target_id: The UUID of the channel on the local instance. Defaults to source_id.
:param download_url: The URL of the Studio instance to import from.
:param editor: The email address of the user you wish to add as an editor, if any.
"""
global log
if logger:
log = logger
else:
log = logging.getLogger(__name__)
# Set up variables for the import process
log.info("\n\n********** STARTING CHANNEL IMPORT **********")
start = datetime.datetime.now()
target_id = target_id or source_id
# Test connection to database
log.info("Connecting to database for channel {}...".format(source_id))
tempf = tempfile.NamedTemporaryFile(suffix=".sqlite3", delete=False)
conn = None
try:
if download_url:
response = requests.get('{}/content/databases/{}.sqlite3'.format(
download_url, source_id))
for chunk in response:
tempf.write(chunk)
else:
filepath = "/".join(
[settings.DB_ROOT, "{}.sqlite3".format(source_id)])
# Check if database exists
if not default_storage.exists(filepath):
raise IOError("The object requested does not exist.")
with default_storage.open(filepath) as fobj:
shutil.copyfileobj(fobj, tempf)
tempf.close()
conn = sqlite3.connect(tempf.name)
cursor = conn.cursor()
# Start by creating channel
log.info("Creating channel...")
channel, root_pk = create_channel(conn, target_id)
if editor:
channel.editors.add(models.User.objects.get(email=editor))
channel.save()
# Create root node
root = models.ContentNode.objects.create(
node_id=root_pk,
title=channel.name,
kind_id=content_kinds.TOPIC,
original_channel_id=target_id,
source_channel_id=target_id,
)
# Create nodes mapping to channel
log.info(" Creating nodes...")
with transaction.atomic():
create_nodes(cursor, target_id, root, download_url=download_url)
# TODO: Handle prerequisites
# Delete the previous tree if it exists
old_previous = channel.previous_tree
if old_previous:
old_previous.parent = get_deleted_chefs_root()
old_previous.title = "Old previous tree for channel {}".format(
channel.pk)
old_previous.save()
# Save tree to target tree
channel.previous_tree = channel.main_tree
channel.main_tree = root
channel.save()
finally:
conn and conn.close()
tempf.close()
os.unlink(tempf.name)
# Print stats
log.info("\n\nChannel has been imported (time: {ms})\n".format(
ms=datetime.datetime.now() - start))
log.info("\n\n********** IMPORT COMPLETE **********\n\n")
