def test_activate_channel(self): previous_tree = self.channel.previous_tree tree(parent=previous_tree) garbage_node = get_deleted_chefs_root() # Previous tree shouldn't be in garbage tree until activate_channel is called self.assertFalse(garbage_node.get_descendants().filter( pk=previous_tree.pk).exists()) activate_channel(self.channel, self.user) garbage_node.refresh_from_db() previous_tree.refresh_from_db() self.channel.refresh_from_db() # We can't use MPTT methods on the deleted chefs tree because we are not running the sort code # for performance reasons, so just do a parent test instead. self.assertTrue(previous_tree.parent == garbage_node) # New previous tree should not be in garbage tree self.assertFalse(self.channel.previous_tree.parent) self.assertNotEqual(garbage_node.tree_id, self.channel.previous_tree.tree_id) child_pk = previous_tree.children.first().pk clean_up_deleted_chefs() self.assertFalse( cc.ContentNode.objects.filter(parent=garbage_node).exists()) self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def test_old_chef_tree(self): # make an actual tree for deletion tests tree(parent=self.channel.chef_tree) chef_tree = self.channel.chef_tree self.assertTrue(chef_tree.get_descendant_count() > 0) garbage_node = get_deleted_chefs_root() self.assertNotEqual(chef_tree, self.channel.staging_tree) # Chef tree shouldn't be in garbage tree until create_channel is called self.assertFalse( garbage_node.get_descendants().filter(pk=chef_tree.pk).exists()) create_channel(self.channel.__dict__, self.user) garbage_node.refresh_from_db() chef_tree.refresh_from_db() self.channel.refresh_from_db() # We can't use MPTT methods to test the deleted chefs tree because we are not running the sort code # for performance reasons, so just do a parent test instead. self.assertEquals(chef_tree.parent.pk, garbage_node.pk) # New staging tree should not be in garbage tree self.assertFalse(self.channel.chef_tree.parent) self.assertNotEqual(garbage_node.tree_id, self.channel.chef_tree.tree_id) child_pk = chef_tree.children.first().pk clean_up_deleted_chefs() self.assertFalse( cc.ContentNode.objects.filter(parent=garbage_node).exists()) self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def test_old_chef_tree(self): # make an actual tree for deletion tests tree(parent=self.channel.chef_tree) chef_tree = self.channel.chef_tree self.assertTrue(chef_tree.get_descendant_count() > 0) garbage_node = get_deleted_chefs_root() self.assertNotEqual(chef_tree, self.channel.staging_tree) # Chef tree shouldn't be in garbage tree until create_channel is called self.assertFalse(garbage_node.get_descendants().filter(pk=chef_tree.pk).exists()) create_channel(self.channel.__dict__, self.user) garbage_node.refresh_from_db() chef_tree.refresh_from_db() self.channel.refresh_from_db() # We can't use MPTT methods to test the deleted chefs tree because we are not running the sort code # for performance reasons, so just do a parent test instead. self.assertEquals(chef_tree.parent.pk, garbage_node.pk) # New staging tree should not be in garbage tree self.assertFalse(self.channel.chef_tree.parent) self.assertNotEqual(garbage_node.tree_id, self.channel.chef_tree.tree_id) child_pk = chef_tree.children.first().pk clean_up_deleted_chefs() self.assertFalse(cc.ContentNode.objects.filter(parent=garbage_node).exists()) self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def test_old_staging_tree(self): staging_tree = self.channel.staging_tree garbage_node = get_deleted_chefs_root() tree(parent=staging_tree) self.assertTrue(staging_tree.get_descendant_count() > 0) # Staging tree shouldn't be in garbage tree until api_commit_channel is called self.assertFalse(garbage_node.get_descendants().filter(pk=staging_tree.pk).exists()) request = self.create_post_request(reverse_lazy('api_finish_channel'), data=json.dumps( {'channel_id': self.channel.pk}), content_type='application/json') response = api_commit_channel(request) self.assertEqual(response.status_code, 200) garbage_node.refresh_from_db() staging_tree.refresh_from_db() self.channel.refresh_from_db() # We can't use MPTT methods on the deleted chefs tree because we are not running the sort code # for performance reasons, so just do a parent test instead. self.assertEqual(staging_tree.parent, garbage_node) # New staging tree should not be in garbage tree self.assertFalse(self.channel.main_tree.parent) self.assertNotEqual(garbage_node.tree_id, self.channel.main_tree.tree_id) child_pk = staging_tree.children.first().pk clean_up_deleted_chefs() self.assertFalse(cc.ContentNode.objects.filter(parent=garbage_node).exists()) self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def activate_channel(channel, user): user.check_channel_space(channel) if channel.previous_tree and channel.previous_tree != channel.main_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with models.ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() channel.previous_tree.parent = garbage_node channel.previous_tree.title = "Previous tree for channel {}".format( channel.pk) channel.previous_tree.save() channel.previous_tree = channel.main_tree channel.main_tree = channel.staging_tree channel.staging_tree = None channel.save() user.staged_files.all().delete() user.set_space_used() change = generate_update_event( channel.id, CHANNEL, { "root_id": channel.main_tree.id, "staging_root_id": None }, ) return change
def test_activate_channel(self): previous_tree = self.channel.previous_tree tree(parent=previous_tree) garbage_node = get_deleted_chefs_root() # Previous tree shouldn't be in garbage tree until activate_channel is called self.assertFalse(garbage_node.get_descendants().filter(pk=previous_tree.pk).exists()) activate_channel(self.channel, self.user) garbage_node.refresh_from_db() previous_tree.refresh_from_db() self.channel.refresh_from_db() # We can't use MPTT methods on the deleted chefs tree because we are not running the sort code # for performance reasons, so just do a parent test instead. self.assertTrue(previous_tree.parent == garbage_node) # New previous tree should not be in garbage tree self.assertFalse(self.channel.previous_tree.parent) self.assertNotEqual(garbage_node.tree_id, self.channel.previous_tree.tree_id) child_pk = previous_tree.children.first().pk clean_up_deleted_chefs() self.assertFalse(cc.ContentNode.objects.filter(parent=garbage_node).exists()) self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def create_channel(channel_data, user): """ Set up channel """ # Set up initial channel channel, isNew = Channel.objects.get_or_create(id=channel_data['id']) # Add user as editor if channel is new or channel has no editors # Otherwise, check if user is an editor if isNew or channel.editors.count() == 0: channel.editors.add(user) elif user not in channel.editors.all(): raise SuspiciousOperation( "User is not authorized to edit this channel") channel.name = channel_data['name'] channel.description = channel_data['description'] channel.thumbnail = channel_data['thumbnail'] channel.deleted = False channel.source_id = channel_data.get('source_id') channel.source_domain = channel_data.get('source_domain') channel.source_url = channel_data.get( 'source_domain') if isNew else channel.source_url channel.ricecooker_version = channel_data.get('ricecooker_version') channel.language_id = channel_data.get('language') # older versions of ricecooker won't be sending this field. if 'tagline' in channel_data: channel.tagline = channel_data['tagline'] old_chef_tree = channel.chef_tree is_published = channel.main_tree is not None and channel.main_tree.published # Set up initial staging tree channel.chef_tree = ContentNode.objects.create( title=channel.name, kind_id=content_kinds.TOPIC, sort_order=get_next_sort_order(), published=is_published, content_id=channel.id, node_id=channel.id, source_id=channel.source_id, source_domain=channel.source_domain, extra_fields={'ricecooker_version': channel.ricecooker_version}, ) channel.chef_tree.save() channel.save() # Delete chef tree if it already exists if old_chef_tree and old_chef_tree != channel.staging_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() old_chef_tree.parent = garbage_node old_chef_tree.title = "Old chef tree for channel {}".format( channel.pk) old_chef_tree.save() return channel # Return new channel
def api_commit_channel(request): """ Commit the channel chef_tree to staging tree to the main tree. This view backs the endpoint `/api/internal/finish_channel` called by ricecooker. """ data = json.loads(request.body) try: channel_id = data['channel_id'] request.user.can_edit(channel_id) obj = Channel.objects.get(pk=channel_id) # Need to rebuild MPTT tree pointers since we used `disable_mptt_updates` ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id) # set original_channel_id and source_channel_id to self since chef tree obj.chef_tree.get_descendants(include_self=True).update( original_channel_id=channel_id, source_channel_id=channel_id) # replace staging_tree with chef_tree old_staging = obj.staging_tree obj.staging_tree = obj.chef_tree obj.chef_tree = None obj.save() # Mark old staging tree for garbage collection if old_staging and old_staging != obj.main_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() old_staging.parent = garbage_node old_staging.title = "Old staging tree for channel {}".format( obj.pk) old_staging.save() # If ricecooker --stage flag used, we're done (skip ACTIVATE step), else # we ACTIVATE the channel, i.e., set the main tree from the staged tree if not data.get('stage'): try: activate_channel(obj, request.user) except PermissionDenied as e: return Response(str(e), status=e.status_code) return Response({ "success": True, "new_channel": obj.pk, }) except (Channel.DoesNotExist, PermissionDenied): return HttpResponseNotFound( "No channel matching: {}".format(channel_id)) except KeyError as e: return HttpResponseBadRequest("Required attribute missing: {}".format( e.message)) except Exception as e: handle_server_error(request) return HttpResponseServerError(content=str(e), reason=str(e))
def create_channel(channel_data, user): """ Set up channel """ # Set up initial channel channel, isNew = Channel.objects.get_or_create(id=channel_data['id']) # Add user as editor if channel is new or channel has no editors # Otherwise, check if user is an editor if isNew or channel.editors.count() == 0: channel.editors.add(user) elif user not in channel.editors.all(): raise SuspiciousOperation("User is not authorized to edit this channel") channel.name = channel_data['name'] channel.description = channel_data['description'] channel.thumbnail = channel_data['thumbnail'] channel.deleted = False channel.source_id = channel_data.get('source_id') channel.source_domain = channel_data.get('source_domain') channel.ricecooker_version = channel_data.get('ricecooker_version') channel.language_id = channel_data.get('language') old_chef_tree = channel.chef_tree is_published = channel.main_tree is not None and channel.main_tree.published # Set up initial staging tree channel.chef_tree = ContentNode.objects.create( title=channel.name, kind_id=content_kinds.TOPIC, sort_order=get_next_sort_order(), published=is_published, content_id=channel.id, node_id=channel.id, source_id=channel.source_id, source_domain=channel.source_domain, extra_fields=json.dumps({'ricecooker_version': channel.ricecooker_version}), ) channel.chef_tree.save() channel.save() # Delete chef tree if it already exists if old_chef_tree and old_chef_tree != channel.staging_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() old_chef_tree.parent = garbage_node old_chef_tree.title = "Old chef tree for channel {}".format(channel.pk) old_chef_tree.save() return channel # Return new channel
def api_commit_channel(request): """ Commit the channel staging tree to the main tree """ data = json.loads(request.body) try: channel_id = data['channel_id'] obj = Channel.objects.get(pk=channel_id) # rebuild MPTT tree for this channel (since we set "disable_mptt_updates", and bulk_create doesn't trigger rebuild signals anyway) ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id) obj.chef_tree.get_descendants(include_self=True).update( original_channel_id=channel_id, source_channel_id=channel_id) old_staging = obj.staging_tree obj.staging_tree = obj.chef_tree obj.chef_tree = None obj.save() # Delete staging tree if it already exists if old_staging and old_staging != obj.main_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() old_staging.parent = garbage_node old_staging.title = "Old staging tree for channel {}".format( obj.pk) old_staging.save() if not data.get( 'stage' ): # If user says to stage rather than submit, skip changing trees at this step try: activate_channel(obj, request.user) except PermissionDenied as e: return HttpResponseForbidden(str(e)) return HttpResponse( json.dumps({ "success": True, "new_channel": obj.pk, })) except KeyError as e: return HttpResponseBadRequest("Required attribute missing: {}".format( e.message)) except Exception as e: handle_server_error(request) return HttpResponseServerError(content=str(e), reason=str(e))
def activate_channel(channel, user): user.check_channel_space(channel) if channel.previous_tree and channel.previous_tree != channel.main_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with models.ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() channel.previous_tree.parent = garbage_node channel.previous_tree.title = "Previous tree for channel {}".format(channel.pk) channel.previous_tree.save() channel.previous_tree = channel.main_tree channel.main_tree = channel.staging_tree channel.staging_tree = None channel.save() user.staged_files.all().delete()
def api_commit_channel(request): """ Commit the channel staging tree to the main tree """ data = json.loads(request.body) try: channel_id = data['channel_id'] obj = Channel.objects.get(pk=channel_id) # rebuild MPTT tree for this channel (since we set "disable_mptt_updates", and bulk_create doesn't trigger rebuild signals anyway) ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id) obj.chef_tree.get_descendants(include_self=True).update(original_channel_id=channel_id, source_channel_id=channel_id) old_staging = obj.staging_tree obj.staging_tree = obj.chef_tree obj.chef_tree = None obj.save() # Delete staging tree if it already exists if old_staging and old_staging != obj.main_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() old_staging.parent = garbage_node old_staging.title = "Old staging tree for channel {}".format(obj.pk) old_staging.save() if not data.get('stage'): # If user says to stage rather than submit, skip changing trees at this step try: activate_channel(obj, request.user) except PermissionDenied as e: return HttpResponseForbidden(str(e)) return HttpResponse(json.dumps({ "success": True, "new_channel": obj.pk, })) except KeyError as e: return HttpResponseBadRequest("Required attribute missing: {}".format(e.message)) except Exception as e: handle_server_error(request) return HttpResponseServerError(content=str(e), reason=str(e))
def test_old_staging_tree(self): staging_tree = self.channel.staging_tree garbage_node = get_deleted_chefs_root() tree(parent=staging_tree) self.assertTrue(staging_tree.get_descendant_count() > 0) # Staging tree shouldn't be in garbage tree until api_commit_channel is called self.assertFalse( garbage_node.get_descendants().filter(pk=staging_tree.pk).exists()) request = self.create_post_request( reverse_lazy("api_finish_channel"), data=json.dumps({"channel_id": self.channel.pk}), content_type="application/json", ) response = api_commit_channel(request) self.assertEqual(response.status_code, 200) garbage_node.refresh_from_db() staging_tree.refresh_from_db() self.channel.refresh_from_db() # We can't use MPTT methods on the deleted chefs tree because we are not running the sort code # for performance reasons, so just do a parent test instead. self.assertEqual(staging_tree.parent, garbage_node) # New staging tree should not be in garbage tree self.assertFalse(self.channel.main_tree.parent) self.assertNotEqual(garbage_node.tree_id, self.channel.main_tree.tree_id) child_pk = staging_tree.children.first().pk clean_up_deleted_chefs() self.assertFalse( cc.ContentNode.objects.filter(parent=garbage_node).exists()) self.assertFalse(cc.ContentNode.objects.filter(pk=child_pk).exists())
def handle(self, *args, **options): try: # Set up variables for restoration process print("\n\n********** STARTING CHANNEL RESTORATION **********") start = datetime.datetime.now() source_id = options['source_id'] target_id = options.get('target') or source_id # Test connection to database print("Connecting to database for channel {}...".format(source_id)) tempf = tempfile.NamedTemporaryFile(suffix=".sqlite3", delete=False) conn = None try: if options.get('download_url'): response = requests.get( '{}/content/databases/{}.sqlite3'.format( options['download_url'], source_id)) for chunk in response: tempf.write(chunk) else: filepath = "/".join( [settings.DB_ROOT, "{}.sqlite3".format(source_id)]) # Check if database exists if not default_storage.exists(filepath): raise IOError("The object requested does not exist.") with default_storage.open(filepath) as fobj: shutil.copyfileobj(fobj, tempf) tempf.close() conn = sqlite3.connect(tempf.name) cursor = conn.cursor() # Start by creating channel print("Creating channel...") channel, root_pk = create_channel(conn, target_id) if options.get('editor'): channel.editors.add( models.User.objects.get(email=options['editor'])) channel.save() # Create root node root = models.ContentNode.objects.create( sort_order=models.get_next_sort_order(), node_id=root_pk, title=channel.name, kind_id=content_kinds.TOPIC, original_channel_id=target_id, source_channel_id=target_id, ) # Create nodes mapping to channel print(" Creating nodes...") with transaction.atomic(): create_nodes(cursor, target_id, root, download_url=options.get('download_url')) # TODO: Handle prerequisites # Delete the previous tree if it exists old_previous = channel.previous_tree if old_previous: old_previous.parent = get_deleted_chefs_root() old_previous.title = "Old previous tree for channel {}".format( channel.pk) old_previous.save() # Save tree to target tree channel.previous_tree = channel.main_tree channel.main_tree = root channel.save() finally: conn and conn.close() tempf.close() os.unlink(tempf.name) # Print stats print("\n\nChannel has been restored (time: {ms})\n".format( ms=datetime.datetime.now() - start)) print("\n\n********** RESTORATION COMPLETE **********\n\n") except EarlyExit as e: logging.warning( "Exited early due to {message}.".format(message=e.message)) self.stdout.write( "You can find your database in {path}".format(path=e.db_path))
def api_commit_channel(request): """ Commit the channel chef_tree to staging tree to the main tree. This view backs the endpoint `/api/internal/finish_channel` called by ricecooker. """ data = json.loads(request.body) try: channel_id = data['channel_id'] request.user.can_edit(channel_id) obj = Channel.objects.get(pk=channel_id) # Need to rebuild MPTT tree pointers since we used `disable_mptt_updates` ContentNode.objects.partial_rebuild(obj.chef_tree.tree_id) # set original_channel_id and source_channel_id to self since chef tree obj.chef_tree.get_descendants(include_self=True).update(original_channel_id=channel_id, source_channel_id=channel_id) # replace staging_tree with chef_tree old_staging = obj.staging_tree obj.staging_tree = obj.chef_tree obj.chef_tree = None obj.save() # Prepare change event indicating a new staging_tree is available event = generate_update_event(channel_id, CHANNEL, { "root_id": obj.main_tree.id, "staging_root_id": obj.staging_tree.id, }) # Mark old staging tree for garbage collection if old_staging and old_staging != obj.main_tree: # IMPORTANT: Do not remove this block, MPTT updating the deleted chefs block could hang the server with ContentNode.objects.disable_mptt_updates(): garbage_node = get_deleted_chefs_root() old_staging.parent = garbage_node old_staging.title = "Old staging tree for channel {}".format(obj.pk) old_staging.save() # Send event (new staging tree or new main tree) to all channel editors for editor in obj.editors.all(): add_event_for_user(editor.id, event) _, task = create_async_task( "get-node-diff", request.user, updated_id=obj.staging_tree.id, original_id=obj.main_tree.id, ) # Send response back to the content integration script return Response({ "success": True, "new_channel": obj.pk, "diff_task_id": task.pk, }) except (Channel.DoesNotExist, PermissionDenied): return HttpResponseNotFound("No channel matching: {}".format(channel_id)) except KeyError: return HttpResponseBadRequest("Required attribute missing from data: {}".format(data)) except Exception as e: handle_server_error(request) return HttpResponseServerError(content=str(e), reason=str(e))
def import_channel(source_id, target_id=None, download_url=None, editor=None, logger=None): """ Import a channel from another Studio instance. This can be used to copy online Studio channels into local machines for development, testing, faster editing, or other purposes. :param source_id: The UUID of the channel to import from the source Studio instance. :param target_id: The UUID of the channel on the local instance. Defaults to source_id. :param download_url: The URL of the Studio instance to import from. :param editor: The email address of the user you wish to add as an editor, if any. """ global log if logger: log = logger else: log = logging.getLogger(__name__) # Set up variables for the import process log.info("\n\n********** STARTING CHANNEL IMPORT **********") start = datetime.datetime.now() target_id = target_id or source_id # Test connection to database log.info("Connecting to database for channel {}...".format(source_id)) tempf = tempfile.NamedTemporaryFile(suffix=".sqlite3", delete=False) conn = None try: if download_url: response = requests.get('{}/content/databases/{}.sqlite3'.format( download_url, source_id)) for chunk in response: tempf.write(chunk) else: filepath = "/".join( [settings.DB_ROOT, "{}.sqlite3".format(source_id)]) # Check if database exists if not default_storage.exists(filepath): raise IOError("The object requested does not exist.") with default_storage.open(filepath) as fobj: shutil.copyfileobj(fobj, tempf) tempf.close() conn = sqlite3.connect(tempf.name) cursor = conn.cursor() # Start by creating channel log.info("Creating channel...") channel, root_pk = create_channel(conn, target_id) if editor: channel.editors.add(models.User.objects.get(email=editor)) channel.save() # Create root node root = models.ContentNode.objects.create( node_id=root_pk, title=channel.name, kind_id=content_kinds.TOPIC, original_channel_id=target_id, source_channel_id=target_id, ) # Create nodes mapping to channel log.info(" Creating nodes...") with transaction.atomic(): create_nodes(cursor, target_id, root, download_url=download_url) # TODO: Handle prerequisites # Delete the previous tree if it exists old_previous = channel.previous_tree if old_previous: old_previous.parent = get_deleted_chefs_root() old_previous.title = "Old previous tree for channel {}".format( channel.pk) old_previous.save() # Save tree to target tree channel.previous_tree = channel.main_tree channel.main_tree = root channel.save() finally: conn and conn.close() tempf.close() os.unlink(tempf.name) # Print stats log.info("\n\nChannel has been imported (time: {ms})\n".format( ms=datetime.datetime.now() - start)) log.info("\n\n********** IMPORT COMPLETE **********\n\n")