Exemple #1
0
 def get_name(static_asset_obj):
     """Method to get the name of the asset."""
     basepath = STATIC_ASSET_BASEPATH.format(
         org=static_asset_obj.course.org,
         course_number=static_asset_obj.course.course_number,
         run=static_asset_obj.course.run,
     )
     return static_asset_obj.asset.name.replace(basepath, '')
Exemple #2
0
 def get_name(static_asset_obj):
     """Method to get the name of the asset."""
     basepath = STATIC_ASSET_BASEPATH.format(
         org=static_asset_obj.course.org,
         course_number=static_asset_obj.course.course_number,
         run=static_asset_obj.course.run,
     )
     return static_asset_obj.asset.name.replace(basepath, '')
Exemple #3
0
def assert_resource_directory(test_case, resources, tempdir):
    """Assert that files are present with correct content."""
    def make_name(resource):
        """Format expected filename."""
        if resource.url_name is not None:
            return "{id}_{url_name}.xml".format(
                id=resource.id,
                url_name=slugify(resource.url_name)[:200],
            )
        else:
            return "{id}.xml".format(id=resource.id)

    # Key is filename, value is count of filenames encountered.
    asset_map = {}

    # Verify that content XML exists in right place.
    for resource in resources:
        type_name = resource.learning_resource_type.name
        abs_path = os.path.join(tempdir, type_name, make_name(resource))
        test_case.assertTrue(os.path.exists(abs_path))
        with open(abs_path) as f:
            test_case.assertEqual(f.read(), resource.content_xml)

        # Verify static asset placement.
        for static_asset in resource.static_assets.all():
            prefix = STATIC_ASSET_BASEPATH.format(
                org=static_asset.course.org,
                course_number=static_asset.course.course_number,
                run=static_asset.course.run
            )
            static_filename = static_asset.asset.name[len(prefix):]

            if static_filename in asset_map:
                count = asset_map[static_filename]
                asset_map[static_filename] += 1
                base, ext = os.path.splitext(static_filename)

                static_filename = "{base}_{count}{ext}".format(
                    base=base,
                    ext=ext,
                    count=count
                )
            else:
                asset_map[static_filename] = 1

            relative_path = os.path.join("static", static_filename)
            abs_path = os.path.join(tempdir, relative_path)
            test_case.assertTrue(os.path.exists(abs_path))
            with open(abs_path, 'rb') as f:
                test_case.assertEqual(f.read(), static_asset.asset.read())
Exemple #4
0
def assert_resource_directory(test_case, resources, tempdir):
    """Assert that files are present with correct content."""
    def make_name(resource):
        """Format expected filename."""
        if resource.url_name is not None:
            return "{id}_{url_name}.xml".format(
                id=resource.id,
                url_name=slugify(resource.url_name)[:200],
            )
        else:
            return "{id}.xml".format(id=resource.id)

    # Key is filename, value is count of filenames encountered.
    asset_map = {}

    # Verify that content XML exists in right place.
    for resource in resources:
        type_name = resource.learning_resource_type.name
        abs_path = os.path.join(tempdir, type_name, make_name(resource))
        test_case.assertTrue(os.path.exists(abs_path))
        with open(abs_path) as f:
            test_case.assertEqual(f.read(), resource.content_xml)

        # Verify static asset placement.
        for static_asset in resource.static_assets.all():
            prefix = STATIC_ASSET_BASEPATH.format(
                org=static_asset.course.org,
                course_number=static_asset.course.course_number,
                run=static_asset.course.run)
            static_filename = static_asset.asset.name[len(prefix):]

            if static_filename in asset_map:
                count = asset_map[static_filename]
                asset_map[static_filename] += 1
                base, ext = os.path.splitext(static_filename)

                static_filename = "{base}_{count}{ext}".format(base=base,
                                                               ext=ext,
                                                               count=count)
            else:
                asset_map[static_filename] = 1

            relative_path = os.path.join("static", static_filename)
            abs_path = os.path.join(tempdir, relative_path)
            test_case.assertTrue(os.path.exists(abs_path))
            with open(abs_path, 'rb') as f:
                test_case.assertEqual(f.read(), static_asset.asset.read())
Exemple #5
0
    def test_duplicate(self):
        """
        Test that exporting resources with same file path
        is reported and handled correctly.
        """

        with TemporaryFile() as temp1:
            with TemporaryFile() as temp2:
                temp1.write(b"first")
                temp2.write(b"second")

                # Create two static assets with the same name but different
                # paths so no renaming is done on storage side.
                static_filename = "iamafile.txt"
                static_path1 = os.path.join(
                    STATIC_ASSET_BASEPATH.format(
                        org=self.resource.course.org,
                        course_number=self.resource.course.course_number,
                        run=self.resource.course.run,
                    ),
                    static_filename
                )

                file1 = File(temp1, name=static_filename)
                file2 = File(temp2, name=static_filename)

                default_storage.delete(static_path1)
                asset1 = create_static_asset(self.resource.course.id, file1)
                self.resource.static_assets.add(asset1)

                course2 = create_course(
                    "org2", self.repo.id, self.resource.course.course_number,
                    self.resource.course.run, self.user.id
                )
                resource2 = create_resource(
                    course=course2,
                    parent=None,
                    resource_type=self.resource.learning_resource_type.name,
                    title=self.resource.title,
                    dpath="",
                    mpath="",
                    content_xml="",
                    url_name=self.resource.url_name
                )
                static_path2 = os.path.join(
                    STATIC_ASSET_BASEPATH.format(
                        org=course2.org,
                        course_number=course2.course_number,
                        run=course2.run,
                    ),
                    static_filename
                )
                default_storage.delete(static_path2)
                asset2 = create_static_asset(resource2.course.id, file2)
                resource2.static_assets.add(asset2)

                # Export the resources. The second static asset should have
                # the number _1 attached to it.
                try:

                    resources = [self.resource, resource2]
                    tempdir, collision = export_resources_to_directory(
                        resources)
                    try:
                        self.assertTrue(collision)
                        assert_resource_directory(self, resources, tempdir)
                    finally:
                        rmtree(tempdir)
                finally:
                    default_storage.delete(asset1.asset.name)
                    default_storage.delete(asset2.asset.name)
Exemple #6
0
def export_resources_to_directory(learning_resources):
    """
    Create files of LearningResource and StaticAsset contents inside directory.

    Args:
        learning_resources (list of learningresources.models.LearningResource):
            LearningResources to export in tarball
    Returns:
        (unicode, bool):
            First item is newly created temp directory with files inside of it.
            Second item is True if a static asset collision was detected.
    """
    def sanitize(url_name):
        """Sanitize title for use in filename."""
        # Limit filename to 200 characters since limit is 256
        # and we have a number and extension too.
        return slugify(url_name)[:200]

    tempdir = mkdtemp()
    collision = False
    type_dirs_created = set()
    try:
        os.mkdir(os.path.join(tempdir, "static"))
        for learning_resource in learning_resources:
            if learning_resource.url_name is not None:
                filename = "{id}_{url_name}.xml".format(
                    id=learning_resource.id,
                    url_name=sanitize(learning_resource.url_name),
                )
            else:
                filename = "{id}.xml".format(id=learning_resource.id)

            type_name = learning_resource.learning_resource_type.name
            if type_name not in type_dirs_created:
                os.mkdir(os.path.join(tempdir, type_name))
                type_dirs_created.add(type_name)
            with open(os.path.join(tempdir, type_name, filename), 'w') as f:
                f.write(learning_resource.content_xml)

            # Output to static directory.
            for static_asset in learning_resource.static_assets.all():
                prefix = STATIC_ASSET_BASEPATH.format(
                    org=static_asset.course.org,
                    course_number=static_asset.course.course_number,
                    run=static_asset.course.run,
                )
                asset_path = static_asset.asset.name
                if asset_path.startswith(prefix):
                    asset_path = asset_path[len(prefix):]

                static_path = os.path.join(tempdir, "static", asset_path)
                static_path, found_collision = _find_unused_path(static_path)
                if found_collision:
                    collision = found_collision

                path, _ = os.path.split(static_path)
                try:
                    os.makedirs(path)
                except OSError as exc:
                    if exc.errno == errno.EEXIST and os.path.isdir(path):
                        pass
                    else:
                        raise
                with open(static_path, 'wb') as outfile:
                    outfile.write(static_asset.asset.read())

        # If we never put anything in static, remove it.
        if not os.listdir(os.path.join(tempdir, "static")):
            rmtree(os.path.join(tempdir, "static"))
    except:
        # Clean up temporary directory since we can't return it anymore.
        rmtree(tempdir)
        raise
    return tempdir, collision
Exemple #7
0
def export_resources_to_directory(learning_resources):
    """
    Create files of LearningResource and StaticAsset contents inside directory.

    Args:
        learning_resources (list of learningresources.models.LearningResource):
            LearningResources to export in tarball
    Returns:
        (unicode, bool):
            First item is newly created temp directory with files inside of it.
            Second item is True if a static asset collision was detected.
    """
    def sanitize(url_name):
        """Sanitize title for use in filename."""
        # Limit filename to 200 characters since limit is 256
        # and we have a number and extension too.
        return slugify(url_name)[:200]

    tempdir = mkdtemp()
    collision = False
    type_dirs_created = set()
    try:
        os.mkdir(os.path.join(tempdir, "static"))
        for learning_resource in learning_resources:
            if learning_resource.url_name is not None:
                filename = "{id}_{url_name}.xml".format(
                    id=learning_resource.id,
                    url_name=sanitize(learning_resource.url_name),
                )
            else:
                filename = "{id}.xml".format(id=learning_resource.id)

            type_name = learning_resource.learning_resource_type.name
            if type_name not in type_dirs_created:
                os.mkdir(os.path.join(tempdir, type_name))
                type_dirs_created.add(type_name)
            with open(os.path.join(tempdir, type_name, filename), 'w') as f:
                f.write(learning_resource.content_xml)

            # Output to static directory.
            for static_asset in learning_resource.static_assets.all():
                prefix = STATIC_ASSET_BASEPATH.format(
                    org=static_asset.course.org,
                    course_number=static_asset.course.course_number,
                    run=static_asset.course.run,
                )
                asset_path = static_asset.asset.name
                if asset_path.startswith(prefix):
                    asset_path = asset_path[len(prefix):]

                static_path = os.path.join(tempdir, "static", asset_path)
                static_path, found_collision = _find_unused_path(static_path)
                if found_collision:
                    collision = found_collision

                path, _ = os.path.split(static_path)
                try:
                    os.makedirs(path)
                except OSError as exc:
                    if exc.errno == errno.EEXIST and os.path.isdir(path):
                        pass
                    else:
                        raise
                with open(static_path, 'wb') as outfile:
                    outfile.write(static_asset.asset.read())

        # If we never put anything in static, remove it.
        if not os.listdir(os.path.join(tempdir, "static")):
            rmtree(os.path.join(tempdir, "static"))
    except:
        # Clean up temporary directory since we can't return it anymore.
        rmtree(tempdir)
        raise
    return tempdir, collision
Exemple #8
0
    def test_duplicate(self):
        """
        Test that exporting resources with same file path
        is reported and handled correctly.
        """

        with TemporaryFile() as temp1:
            with TemporaryFile() as temp2:
                temp1.write(b"first")
                temp2.write(b"second")

                # Create two static assets with the same name but different
                # paths so no renaming is done on storage side.
                static_filename = "iamafile.txt"
                static_path1 = os.path.join(
                    STATIC_ASSET_BASEPATH.format(
                        org=self.resource.course.org,
                        course_number=self.resource.course.course_number,
                        run=self.resource.course.run,
                    ), static_filename)

                file1 = File(temp1, name=static_filename)
                file2 = File(temp2, name=static_filename)

                default_storage.delete(static_path1)
                asset1 = create_static_asset(self.resource.course.id, file1)
                self.resource.static_assets.add(asset1)

                course2 = create_course("org2", self.repo.id,
                                        self.resource.course.course_number,
                                        self.resource.course.run, self.user.id)
                resource2 = create_resource(
                    course=course2,
                    parent=None,
                    resource_type=self.resource.learning_resource_type.name,
                    title=self.resource.title,
                    dpath="",
                    mpath="",
                    content_xml="",
                    url_name=self.resource.url_name)
                static_path2 = os.path.join(
                    STATIC_ASSET_BASEPATH.format(
                        org=course2.org,
                        course_number=course2.course_number,
                        run=course2.run,
                    ), static_filename)
                default_storage.delete(static_path2)
                asset2 = create_static_asset(resource2.course.id, file2)
                resource2.static_assets.add(asset2)

                # Export the resources. The second static asset should have
                # the number _1 attached to it.
                try:

                    resources = [self.resource, resource2]
                    tempdir, collision = export_resources_to_directory(
                        resources)
                    try:
                        self.assertTrue(collision)
                        assert_resource_directory(self, resources, tempdir)
                    finally:
                        rmtree(tempdir)
                finally:
                    default_storage.delete(asset1.asset.name)
                    default_storage.delete(asset2.asset.name)