Beispiel #1
0
    def save(self, *args, **kwargs):
        from catalogue.utils import ExistingFile, remove_zip

        try:
            old = BookMedia.objects.get(pk=self.pk)
        except BookMedia.DoesNotExist:
            old = None
        else:
            # if name changed, change the file name, too
            if slughifi(self.name) != slughifi(old.name):
                self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)

        super(BookMedia, self).save(*args, **kwargs)

        # remove the zip package for book with modified media
        if old:
            remove_zip("%s_%s" % (old.book.slug, old.type))
        remove_zip("%s_%s" % (self.book.slug, self.type))

        extra_info = self.extra_info
        if isinstance(extra_info, basestring):
            # Walkaround for weird jsonfield 'no-decode' optimization.
            extra_info = json.loads(extra_info)
        extra_info.update(self.read_meta())
        self.extra_info = extra_info
        self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
        return super(BookMedia, self).save(*args, **kwargs)
Beispiel #2
0
    def save(self, *args, **kwargs):
        from fnpdjango.utils.text.slughifi import slughifi
        from catalogue.utils import ExistingFile, remove_zip

        try:
            old = BookMedia.objects.get(pk=self.pk)
        except BookMedia.DoesNotExist:
            old = None
        else:
            # if name changed, change the file name, too
            if slughifi(self.name) != slughifi(old.name):
                self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)

        super(BookMedia, self).save(*args, **kwargs)

        # remove the zip package for book with modified media
        if old:
            remove_zip("%s_%s" % (old.book.slug, old.type))
        remove_zip("%s_%s" % (self.book.slug, self.type))

        extra_info = self.extra_info
        if isinstance(extra_info, basestring):
            # Walkaround for weird jsonfield 'no-decode' optimization.
            extra_info = json.loads(extra_info)
        extra_info.update(self.read_meta())
        self.extra_info = extra_info
        self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
        return super(BookMedia, self).save(*args, **kwargs)
Beispiel #3
0
def get_dynamic_path(media, filename, ext=None, maxlen=100):
    from fnpdjango.utils.text.slughifi import slughifi

    # how to put related book's slug here?
    if not ext:
        # BookMedia case
        ext = media.formats[media.type].ext
    if media is None or not media.name:
        name = slughifi(filename.split(".")[0])
    else:
        name = slughifi(media.name)
    return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
Beispiel #4
0
 def tags_from_info(info):
     from fnpdjango.utils.text.slughifi import slughifi
     from sortify import sortify
     meta_tags = []
     categories = (('kinds', 'kind'), ('genres', 'genre'),
                   ('authors', 'author'), ('epochs', 'epoch'))
     for field_name, category in categories:
         try:
             tag_names = getattr(info, field_name)
         except:
             try:
                 tag_names = [getattr(info, category)]
             except:
                 # For instance, Pictures do not have 'genre' field.
                 continue
         for tag_name in tag_names:
             tag_sort_key = tag_name
             if category == 'author':
                 tag_sort_key = tag_name.last_name
                 tag_name = tag_name.readable()
             tag, created = Tag.objects.get_or_create(
                 slug=slughifi(tag_name), category=category)
             if created:
                 tag.name = tag_name
                 tag.sort_key = sortify(tag_sort_key.lower())
                 tag.save()
             meta_tags.append(tag)
     return meta_tags
 def tags_from_info(info):
     from fnpdjango.utils.text.slughifi import slughifi
     from sortify import sortify
     meta_tags = []
     categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
     for field_name, category in categories:
         try:
             tag_names = getattr(info, field_name)
         except:
             try:
                 tag_names = [getattr(info, category)]
             except:
                 # For instance, Pictures do not have 'genre' field.
                 continue
         for tag_name in tag_names:
             tag_sort_key = tag_name
             if category == 'author':
                 tag_sort_key = tag_name.last_name
                 tag_name = tag_name.readable()
             tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
             if created:
                 tag.name = tag_name
                 tag.sort_key = sortify(tag_sort_key.lower())
                 tag.save()
             meta_tags.append(tag)
     return meta_tags
Beispiel #6
0
def info_args(title, language=None):
    """ generate some keywords for comfortable BookInfoCreation  """
    slug = unicode(slughifi(title))
    if language is None:
        language = u'pol'
    return {
        'title': unicode(title),
        'url': WLURI.from_slug(slug),
        'about': u"http://wolnelektury.pl/example/URI/%s" % slug,
        'language': language,
    }
Beispiel #7
0
def info_args(title, language=None):
    """ generate some keywords for comfortable BookInfoCreation  """
    slug = unicode(slughifi(title))
    if language is None:
        language = u'pol'
    return {
        'title': unicode(title),
        'url': WLURI.from_slug(slug),
        'about': u"http://wolnelektury.pl/example/URI/%s" % slug,
        'language': language,
    }
Beispiel #8
0
    def save(self, *args, **kwargs):
        from catalogue.utils import ExistingFile, remove_zip

        parts_count = BookMedia.objects.filter(book=self.book,
                                               type=self.type).count()
        if parts_count == 1:
            self.name = self.book.pretty_title()
        else:
            no = ('%02d' if parts_count < 100 else '%03d') % self.index
            self.name = '%s. %s' % (no, self.book.pretty_title())
            if self.part_name:
                self.name += ', ' + self.part_name

        try:
            old = BookMedia.objects.get(pk=self.pk)
        except BookMedia.DoesNotExist:
            old = None
        else:
            # if name changed, change the file name, too
            if slughifi(self.name) != slughifi(old.name):
                self.file.save(None,
                               ExistingFile(self.file.path),
                               save=False,
                               leave=True)

        super(BookMedia, self).save(*args, **kwargs)

        # remove the zip package for book with modified media
        if old:
            remove_zip("%s_%s" % (old.book.slug, old.type))
        remove_zip("%s_%s" % (self.book.slug, self.type))

        extra_info = self.extra_info
        if isinstance(extra_info, basestring):
            # Walkaround for weird jsonfield 'no-decode' optimization.
            extra_info = json.loads(extra_info)
        extra_info.update(self.read_meta())
        self.extra_info = extra_info
        self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
        return super(BookMedia, self).save(*args, **kwargs)
Beispiel #9
0
def gallery(slug, text):
    result = {}

    m = re.match(META_REGEX, text)
    if m:
        for line in m.group(1).split('\n'):
            try:
                k, v = line.split(':', 1)
                result[k.strip()] = v.strip()
            except ValueError:
                continue

    gallery = result.get('gallery', slughifi(slug))

    if gallery.startswith('/'):
        gallery = os.path.basename(gallery)

    return gallery
Beispiel #10
0
    def tags_from_info(info):
        from fnpdjango.utils.text.slughifi import slughifi
        from sortify import sortify
        meta_tags = []
        categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
        for field_name, category in categories:
            try:
                tag_names = getattr(info, field_name)
            except KeyError:
                try:
                    tag_names = [getattr(info, category)]
                except KeyError:
                    # For instance, Pictures do not have 'genre' field.
                    continue
            for tag_name in tag_names:
                lang = getattr(tag_name, 'lang', settings.LANGUAGE_CODE)
                tag_sort_key = tag_name
                if category == 'author':
                    tag_sort_key = ' '.join((tag_name.last_name,) + tag_name.first_names)
                    tag_name = tag_name.readable()
                if lang == settings.LANGUAGE_CODE:
                    # Allow creating new tag, if it's in default language.
                    tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
                    if created:
                        tag_name = unicode(tag_name)
                        tag.name = tag_name
                        setattr(tag, "name_%s" % lang, tag_name)
                        tag.sort_key = sortify(tag_sort_key.lower())
                        tag.save()

                    meta_tags.append(tag)
                else:
                    # Ignore unknown tags in non-default languages.
                    try:
                        tag = Tag.objects.get(category=category, **{"name_%s" % lang: tag_name})
                    except Tag.DoesNotExist:
                        pass
                    else:
                        meta_tags.append(tag)
        return meta_tags
Beispiel #11
0
    def tags_from_info(info):
        from fnpdjango.utils.text.slughifi import slughifi
        from sortify import sortify
        meta_tags = []
        categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
        for field_name, category in categories:
            try:
                tag_names = getattr(info, field_name)
            except:
                try:
                    tag_names = [getattr(info, category)]
                except:
                    # For instance, Pictures do not have 'genre' field.
                    continue
            for tag_name in tag_names:
                lang = getattr(tag_name, 'lang', settings.LANGUAGE_CODE)
                tag_sort_key = tag_name
                if category == 'author':
                    tag_sort_key = tag_name.last_name
                    tag_name = tag_name.readable()
                if lang == settings.LANGUAGE_CODE:
                    # Allow creating new tag, if it's in default language.
                    tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
                    if created:
                        tag_name = unicode(tag_name)
                        tag.name = tag_name
                        setattr(tag, "name_%s" % lang, tag_name)
                        tag.sort_key = sortify(tag_sort_key.lower())
                        tag.save()

                    meta_tags.append(tag)
                else:
                    # Ignore unknown tags in non-default languages.
                    try:
                        tag = Tag.objects.get(category=category, **{"name_%s" % lang: tag_name})
                    except Tag.DoesNotExist:
                        pass
                    else:
                        meta_tags.append(tag)
        return meta_tags
Beispiel #12
0
    def build(self, fieldfile):
        from django.core.files.base import ContentFile
        from fnpdjango.utils.text.slughifi import slughifi
        from sortify import sortify
        from librarian import html
        from catalogue.models import Fragment, Tag

        book = fieldfile.instance

        html_output = self.transform(book.wldocument(parse_dublincore=False),
                                     fieldfile)

        # Delete old fragments, create from scratch if necessary.
        book.fragments.all().delete()

        if html_output:
            meta_tags = list(
                book.tags.filter(category__in=('author', 'epoch', 'genre',
                                               'kind')))

            lang = book.language
            lang = LANGUAGES_3TO2.get(lang, lang)
            if lang not in [ln[0] for ln in settings.LANGUAGES]:
                lang = None

            fieldfile.save(None,
                           ContentFile(html_output.get_string()),
                           save=False)
            type(book).objects.filter(pk=book.pk).update(
                **{fieldfile.field.attname: fieldfile})

            # Extract fragments
            closed_fragments, open_fragments = html.extract_fragments(
                fieldfile.path)
            for fragment in closed_fragments.values():
                try:
                    theme_names = [
                        s.strip() for s in fragment.themes.split(',')
                    ]
                except AttributeError:
                    continue
                themes = []
                for theme_name in theme_names:
                    if not theme_name:
                        continue
                    if lang == settings.LANGUAGE_CODE:
                        # Allow creating themes if book in default language.
                        tag, created = Tag.objects.get_or_create(
                            slug=slughifi(theme_name), category='theme')
                        if created:
                            tag.name = theme_name
                            setattr(tag, "name_%s" % lang, theme_name)
                            tag.sort_key = sortify(theme_name.lower())
                            tag.save()
                        themes.append(tag)
                    elif lang is not None:
                        # Don't create unknown themes in non-default languages.
                        try:
                            tag = Tag.objects.get(
                                category='theme',
                                **{"name_%s" % lang: theme_name})
                        except Tag.DoesNotExist:
                            pass
                        else:
                            themes.append(tag)
                if not themes:
                    continue

                text = fragment.to_string()
                short_text = truncate_html_words(text, 15)
                if text == short_text:
                    short_text = ''
                new_fragment = Fragment.objects.create(anchor=fragment.id,
                                                       book=book,
                                                       text=text,
                                                       short_text=short_text)

                new_fragment.save()
                new_fragment.tags = set(meta_tags + themes)
            book.html_built.send(sender=type(self), instance=book)
            return True
        return False
Beispiel #13
0
def _file_upload_to(i, _n):
    return 'book/%(ext)s/%(name)s.%(ext)s' % {
        'ext': i.ext(),
        'name': slughifi(i.name)
    }
Beispiel #14
0
def _file_upload_to(i, _n):
    return 'book/%(ext)s/%(name)s.%(ext)s' % {
            'ext': i.ext(), 'name': slughifi(i.name)}
Beispiel #15
0
    def build(self, fieldfile):
        from django.core.files.base import ContentFile
        from fnpdjango.utils.text.slughifi import slughifi
        from sortify import sortify
        from librarian import html
        from catalogue.models import Fragment, Tag

        book = fieldfile.instance

        html_output = self.transform(
                        book.wldocument(parse_dublincore=False),
                        fieldfile)

        # Delete old fragments, create from scratch if necessary.
        book.fragments.all().delete()

        if html_output:
            meta_tags = list(book.tags.filter(
                category__in=('author', 'epoch', 'genre', 'kind')))

            lang = book.language
            lang = LANGUAGES_3TO2.get(lang, lang)
            if lang not in [ln[0] for ln in settings.LANGUAGES]:
                lang = None

            fieldfile.save(None, ContentFile(html_output.get_string()),
                    save=False)
            type(book).objects.filter(pk=book.pk).update(**{
                fieldfile.field.attname: fieldfile
            })

            # Extract fragments
            closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
            for fragment in closed_fragments.values():
                try:
                    theme_names = [s.strip() for s in fragment.themes.split(',')]
                except AttributeError:
                    continue
                themes = []
                for theme_name in theme_names:
                    if not theme_name:
                        continue
                    if lang == settings.LANGUAGE_CODE:
                        # Allow creating themes if book in default language.
                        tag, created = Tag.objects.get_or_create(
                                            slug=slughifi(theme_name),
                                            category='theme')
                        if created:
                            tag.name = theme_name
                            setattr(tag, "name_%s" % lang, theme_name)
                            tag.sort_key = sortify(theme_name.lower())
                            tag.save()
                        themes.append(tag)
                    elif lang is not None:
                        # Don't create unknown themes in non-default languages.
                        try:
                            tag = Tag.objects.get(category='theme',
                                    **{"name_%s" % lang: theme_name})
                        except Tag.DoesNotExist:
                            pass
                        else:
                            themes.append(tag)
                if not themes:
                    continue

                text = fragment.to_string()
                short_text = truncate_html_words(text, 15)
                if text == short_text:
                    short_text = ''
                new_fragment = Fragment.objects.create(anchor=fragment.id,
                        book=book, text=text, short_text=short_text)

                new_fragment.save()
                new_fragment.tags = set(meta_tags + themes)
            book.html_built.send(sender=type(self), instance=book)
            return True
        return False
Beispiel #16
0
def migrate_file_from_hg(orm, fname, entry):
    fname = urlunquote(fname)
    print fname
    if fname.endswith('.xml'):
        fname = fname[:-4]
    title = file_to_title(fname)
    fname = slughifi(fname)

    # create all the needed objects
    # what if it already exists?
    book = orm.Book.objects.create(
        title=title,
        slug=fname)
    chunk = orm.Chunk.objects.create(
        book=book,
        number=1,
        slug='1')
    try:
        chunk.stage = orm.ChunkTag.objects.order_by('ordering')[0]
    except IndexError:
        chunk.stage = None

    maxrev = entry.filerev()
    gallery_link = None

    # this will fail if directory exists
    os.makedirs(os.path.join(settings.CATALOGUE_REPO_PATH, str(chunk.pk)))

    for rev in xrange(maxrev + 1):
        fctx = entry.filectx(rev)
        data = fctx.data()
        gallery_link = gallery(fname, data)
        data = plain_text(data)

        # get tags from description
        description = fctx.description().decode("utf-8", 'replace')
        tags = STAGE_TAGS_RE.findall(description)
        tags = [orm.ChunkTag.objects.get(slug=slug.strip()) for slug in tags]

        if tags:
            max_ordering = max(tags, key=lambda x: x.ordering).ordering
            try:
                chunk.stage = orm.ChunkTag.objects.filter(ordering__gt=max_ordering).order_by('ordering')[0]
            except IndexError:
                chunk.stage = None

        description = STAGE_TAGS_RE.sub('', description)

        author = author_name = author_email = None
        author_desc = fctx.user().decode("utf-8", 'replace')
        m = AUTHOR_RE.match(author_desc)
        if m:
            try:
                author = orm['auth.User'].objects.get(username=m.group(1), email=m.group(2))
            except orm['auth.User'].DoesNotExist:
                author_name = m.group(1)
                author_email = m.group(2)
        else:
            author_name = author_desc

        head = orm.ChunkChange.objects.create(
            tree=chunk,
            revision=rev + 1,
            created_at=datetime.datetime.fromtimestamp(fctx.date()[0]),
            description=description,
            author=author,
            author_name=author_name,
            author_email=author_email,
            parent=chunk.head
            )

        path = "%d/%d" % (chunk.pk, head.pk)
        abs_path = os.path.join(settings.CATALOGUE_REPO_PATH, path)
        f = open(abs_path, 'wb')
        f.write(compress(data))
        f.close()
        head.data = path

        head.tags = tags
        head.save()

        chunk.head = head

    chunk.save()
    if gallery_link:
        book.gallery = gallery_link
        book.save()
Beispiel #17
0
    def from_xml_file(cls, xml_file, image_file=None, overwrite=False):
        """
        Import xml and it's accompanying image file.
        If image file is missing, it will be fetched by librarian.picture.ImageStore
        which looks for an image file in the same directory the xml is, with extension matching
        its mime type.
        """
        from sortify import sortify
        from django.core.files import File
        from librarian.picture import WLPicture, ImageStore
        close_xml_file = False
        close_image_file = False
        # class SimpleImageStore(object):
        #     def path(self_, slug, mime_type):
        #         """Returns the image file. Ignores slug ad mime_type."""
        #         return image_file

        if image_file is not None and not isinstance(image_file, File):
            image_file = File(open(image_file))
            close_image_file = True

        if not isinstance(xml_file, File):
            xml_file = File(open(xml_file))
            close_xml_file = True

        try:
            # use librarian to parse meta-data
            picture_xml = WLPicture.from_file(
                xml_file,
                image_store=ImageStore(picture_storage.path('images')))
            # image_store=SimpleImageStore

            picture, created = Picture.objects.get_or_create(
                slug=picture_xml.slug)
            if not created and not overwrite:
                raise Picture.AlreadyExists('Picture %s already exists' %
                                            picture_xml.slug)

            picture.title = picture_xml.picture_info.title

            motif_tags = set()
            for part in picture_xml.partiter():
                for motif in part['themes']:
                    tag, created = catalogue.models.Tag.objects.get_or_create(
                        slug=slughifi(motif), category='theme')
                    if created:
                        tag.name = motif
                        tag.sort_key = sortify(tag.name)
                        tag.save()
                    motif_tags.add(tag)

            picture.tags = catalogue.models.Tag.tags_from_info(picture_xml.picture_info) + \
                list(motif_tags)

            if image_file is not None:
                img = image_file
            else:
                img = picture_xml.image_file()

            # FIXME: hardcoded extension
            picture.image_file.save(path.basename(picture_xml.image_path),
                                    File(img))

            picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
            picture.save()
        finally:
            if close_xml_file:
                xml_file.close()
            if close_image_file:
                image_file.close()
        return picture
Beispiel #18
0
    def from_xml_file(cls, xml_file, image_file=None, overwrite=False):
        """
        Import xml and it's accompanying image file.
        If image file is missing, it will be fetched by librarian.picture.ImageStore
        which looks for an image file in the same directory the xml is, with extension matching
        its mime type.
        """
        from sortify import sortify
        from django.core.files import File
        from librarian.picture import WLPicture, ImageStore

        close_xml_file = False
        close_image_file = False
        # class SimpleImageStore(object):
        #     def path(self_, slug, mime_type):
        #         """Returns the image file. Ignores slug ad mime_type."""
        #         return image_file

        if image_file is not None and not isinstance(image_file, File):
            image_file = File(open(image_file))
            close_image_file = True

        if not isinstance(xml_file, File):
            xml_file = File(open(xml_file))
            close_xml_file = True

        try:
            # use librarian to parse meta-data
            picture_xml = WLPicture.from_file(xml_file, image_store=ImageStore(picture_storage.path("images")))
            # image_store=SimpleImageStore

            picture, created = Picture.objects.get_or_create(slug=picture_xml.slug)
            if not created and not overwrite:
                raise Picture.AlreadyExists("Picture %s already exists" % picture_xml.slug)

            picture.title = picture_xml.picture_info.title

            motif_tags = set()
            for part in picture_xml.partiter():
                for motif in part["themes"]:
                    tag, created = catalogue.models.Tag.objects.get_or_create(slug=slughifi(motif), category="theme")
                    if created:
                        tag.name = motif
                        tag.sort_key = sortify(tag.name)
                        tag.save()
                    motif_tags.add(tag)

            picture.tags = catalogue.models.Tag.tags_from_info(picture_xml.picture_info) + list(motif_tags)

            if image_file is not None:
                img = image_file
            else:
                img = picture_xml.image_file()

            # FIXME: hardcoded extension
            picture.image_file.save(path.basename(picture_xml.image_path), File(img))

            picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
            picture.save()
        finally:
            if close_xml_file:
                xml_file.close()
            if close_image_file:
                image_file.close()
        return picture
    def handle(self, *redmine_csv, **options):

        self.style = color_style()

        redakcja = options.get('redakcja')
        verbose = options.get('verbose')
        force = options.get('force')

        if not redmine_csv:
            if verbose:
                print "Using default Redmine CSV URL:", REDMINE_CSV
            redmine_csv = REDMINE_CSV

        # Start transaction management.
        transaction.commit_unless_managed()
        transaction.enter_transaction_management()
        transaction.managed(True)

        redakcja_link = re.compile(re.escape(redakcja) + r'([-_.:?&%/a-zA-Z0-9]*)')

        all_tickets = 0
        all_chunks = 0
        done_tickets = 0
        done_chunks = 0
        empty_users = 0
        unknown_users = {}
        unknown_books = []
        forced = []

        if verbose:
            print 'Downloading CSV file'
        for r in csv.reader(urllib2.urlopen(redmine_csv)):
            if r[0] == '#':
                continue
            all_tickets += 1

            username = r[6]
            if not username:
                if verbose:
                    print "Empty user, skipping"
                empty_users += 1
                continue

            first_name, last_name = unicode(username, 'utf-8').rsplit(u' ', 1)
            try:
                user = User.objects.get(first_name=first_name, last_name=last_name)
            except User.DoesNotExist:
                print self.style.ERROR('Unknown user: '******'utf-8', 'ignore')
                if fname.endswith('.xml'):
                    fname = fname[:-4]
                fname = fname.replace(' ', '_')
                fname = slughifi(fname)

                chunks = Chunk.objects.filter(book__slug=fname)
                if not chunks:
                    print self.style.ERROR('Unknown book: ' + fname)
                    unknown_books.append(fname)
                    continue
                all_chunks += chunks.count()

                for chunk in chunks:
                    if chunk.user:
                        if chunk.user == user:
                            continue
                        else:
                            forced.append((chunk, chunk.user, user))
                            if force:
                                print self.style.WARNING(
                                    '%s assigned to %s, forcing change to %s.' %
                                    (chunk.pretty_name(), chunk.user, user))
                            else:
                                print self.style.WARNING(
                                    '%s assigned to %s not to %s, skipping.' %
                                    (chunk.pretty_name(), chunk.user, user))
                                continue
                    chunk.user = user
                    chunk.save()
                    ticket_done = True
                    done_chunks += 1

            if ticket_done:
                done_tickets += 1


        # Print results
        print
        print "Results:"
        print "Assignments imported from %d/%d tickets to %d/%d relevalt chunks." % (
                done_tickets, all_tickets, done_chunks, all_chunks)
        if empty_users:
            print "%d tickets were unassigned." % empty_users
        if forced:
            print "%d assignments conficts (%s):" % (
                len(forced), "changed" if force else "left")
            for chunk, orig, user in forced:
                print "  %s: \t%s \t->  %s" % (
                    chunk.pretty_name(), orig.username, user.username)
        if unknown_books:
            print "%d unknown books:" % len(unknown_books)
            for fname in unknown_books:
                print "  %s" % fname
        if unknown_users:
            print "%d unknown users:" % len(unknown_users)
            for name in unknown_users:
                print "  %s (%d tickets)" % (name, unknown_users[name])
        print


        transaction.commit()
        transaction.leave_transaction_management()
Beispiel #20
0
    def build(self, fieldfile):
        from django.core.files.base import ContentFile
        from fnpdjango.utils.text.slughifi import slughifi
        from sortify import sortify
        from librarian import html
        from catalogue.models import Fragment, Tag

        book = fieldfile.instance

        meta_tags = list(
            book.tags.filter(category__in=('author', 'epoch', 'genre',
                                           'kind')))
        book_tag = book.book_tag()

        html_output = self.transform(book.wldocument(parse_dublincore=False),
                                     fieldfile)
        if html_output:
            fieldfile.save(None,
                           ContentFile(html_output.get_string()),
                           save=False)
            type(book).objects.filter(pk=book.pk).update(
                **{fieldfile.field.attname: fieldfile})

            # get ancestor l-tags for adding to new fragments
            ancestor_tags = []
            p = book.parent
            while p:
                ancestor_tags.append(p.book_tag())
                p = p.parent

            # Delete old fragments and create them from scratch
            book.fragments.all().delete()
            # Extract fragments
            closed_fragments, open_fragments = html.extract_fragments(
                fieldfile.path)
            for fragment in closed_fragments.values():
                try:
                    theme_names = [
                        s.strip() for s in fragment.themes.split(',')
                    ]
                except AttributeError:
                    continue
                themes = []
                for theme_name in theme_names:
                    if not theme_name:
                        continue
                    tag, created = Tag.objects.get_or_create(
                        slug=slughifi(theme_name), category='theme')
                    if created:
                        tag.name = theme_name
                        tag.sort_key = sortify(theme_name.lower())
                        tag.save()
                    themes.append(tag)
                if not themes:
                    continue

                text = fragment.to_string()
                short_text = truncate_html_words(text, 15)
                if text == short_text:
                    short_text = ''
                new_fragment = Fragment.objects.create(anchor=fragment.id,
                                                       book=book,
                                                       text=text,
                                                       short_text=short_text)

                new_fragment.save()
                new_fragment.tags = set(meta_tags + themes + [book_tag] +
                                        ancestor_tags)
            book.html_built.send(sender=book)
            return True
        return False
Beispiel #21
0
    def from_xml_file(cls, xml_file, image_file=None, image_store=None, overwrite=False):
        """
        Import xml and it's accompanying image file.
        If image file is missing, it will be fetched by librarian.picture.ImageStore
        which looks for an image file in the same directory the xml is, with extension matching
        its mime type.
        """
        from sortify import sortify
        from django.core.files import File
        from librarian.picture import WLPicture, ImageStore
        close_xml_file = False
        close_image_file = False

        if image_file is not None and not isinstance(image_file, File):
            image_file = File(open(image_file))
            close_image_file = True

        if not isinstance(xml_file, File):
            xml_file = File(open(xml_file))
            close_xml_file = True

        with transaction.atomic():
            # use librarian to parse meta-data
            if image_store is None:
                image_store = ImageStore(picture_storage.path('images'))
            picture_xml = WLPicture.from_file(xml_file, image_store=image_store)

            picture, created = Picture.objects.get_or_create(slug=picture_xml.slug[:120])
            if not created and not overwrite:
                raise Picture.AlreadyExists('Picture %s already exists' % picture_xml.slug)

            picture.areas.all().delete()
            picture.title = unicode(picture_xml.picture_info.title)
            picture.extra_info = picture_xml.picture_info.to_dict()

            picture_tags = set(catalogue.models.Tag.tags_from_info(picture_xml.picture_info))
            motif_tags = set()
            thing_tags = set()

            area_data = {'themes': {}, 'things': {}}

            # Treat all names in picture XML as in default language.
            lang = settings.LANGUAGE_CODE

            for part in picture_xml.partiter():
                if picture_xml.frame:
                    c = picture_xml.frame[0]
                    part['coords'] = [[p[0] - c[0], p[1] - c[1]] for p in part['coords']]
                if part.get('object', None) is not None:
                    _tags = set()
                    for objname in part['object'].split(','):
                        objname = objname.strip().capitalize()
                        tag, created = catalogue.models.Tag.objects.get_or_create(
                            slug=slughifi(objname), category='thing')
                        if created:
                            tag.name = objname
                            setattr(tag, 'name_%s' % lang, tag.name)
                            tag.sort_key = sortify(tag.name)
                            tag.save()
                        # thing_tags.add(tag)
                        area_data['things'][tag.slug] = {
                            'object': objname,
                            'coords': part['coords'],
                            }

                        _tags.add(tag)
                    area = PictureArea.rectangle(picture, 'thing', part['coords'])
                    area.save()
                    area.tags = _tags
                else:
                    _tags = set()
                    for motifs in part['themes']:
                        for motif in motifs.split(','):
                            tag, created = catalogue.models.Tag.objects.get_or_create(
                                slug=slughifi(motif), category='theme')
                            if created:
                                tag.name = motif
                                tag.sort_key = sortify(tag.name)
                                tag.save()
                            # motif_tags.add(tag)
                            _tags.add(tag)
                            area_data['themes'][tag.slug] = {
                                'theme': motif,
                                'coords': part['coords']
                                }

                    logging.debug("coords for theme: %s" % part['coords'])
                    area = PictureArea.rectangle(picture, 'theme', part['coords'])
                    area.save()
                    area.tags = _tags.union(picture_tags)

            picture.tags = picture_tags.union(motif_tags).union(thing_tags)
            picture.areas_json = area_data

            if image_file is not None:
                img = image_file
            else:
                img = picture_xml.image_file()

            modified = cls.crop_to_frame(picture_xml, img)
            modified = cls.add_source_note(picture_xml, modified)

            picture.width, picture.height = modified.size

            modified_file = StringIO()
            modified.save(modified_file, format='JPEG', quality=95)
            # FIXME: hardcoded extension - detect from DC format or orginal filename
            picture.image_file.save(path.basename(picture_xml.image_path), File(modified_file))

            picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
            picture.save()
            tasks.generate_picture_html(picture.id)

        if close_xml_file:
            xml_file.close()
        if close_image_file:
            image_file.close()

        return picture
Beispiel #22
0
def upload(request):
    if request.method == "POST":
        form = forms.DocumentsUploadForm(request.POST, request.FILES)
        if form.is_valid():
            from fnpdjango.utils.text.slughifi import slughifi

            if request.user.is_authenticated():
                creator = request.user
            else:
                creator = None

            zip = form.cleaned_data['zip']
            skipped_list = []
            ok_list = []
            error_list = []
            slugs = {}
            existing = [book.slug for book in Book.objects.all()]
            for filename in zip.namelist():
                if filename[-1] == '/':
                    continue
                title = os.path.basename(filename)[:-4]
                slug = slughifi(title)
                if not (slug and filename.endswith('.xml')):
                    skipped_list.append(filename)
                elif slug in slugs:
                    error_list.append((filename, slug, _('Slug already used for %s' % slugs[slug])))
                elif slug in existing:
                    error_list.append((filename, slug, _('Slug already used in repository.')))
                else:
                    try:
                        zip.read(filename).decode('utf-8') # test read
                        ok_list.append((filename, slug, title))
                    except UnicodeDecodeError:
                        error_list.append((filename, title, _('File should be UTF-8 encoded.')))
                    slugs[slug] = filename

            if not error_list:
                for filename, slug, title in ok_list:
                    book = Book.create(
                        text=zip.read(filename).decode('utf-8'),
                        creator=creator,
                        slug=slug,
                        title=title,
                    )

            return render(request, "catalogue/document_upload.html", {
                "form": form,
                "ok_list": ok_list,
                "skipped_list": skipped_list,
                "error_list": error_list,

                "logout_to": '/',
            })
    else:
        form = forms.DocumentsUploadForm()

    return render(request, "catalogue/document_upload.html", {
        "form": form,

        "logout_to": '/',
    })
    def build(self, fieldfile):
        from django.core.files.base import ContentFile
        from fnpdjango.utils.text.slughifi import slughifi
        from sortify import sortify
        from librarian import html
        from catalogue.models import Fragment, Tag

        book = fieldfile.instance

        meta_tags = list(book.tags.filter(
            category__in=('author', 'epoch', 'genre', 'kind')))
        book_tag = book.book_tag()

        html_output = self.transform(
                        book.wldocument(parse_dublincore=False),
                        fieldfile)
        if html_output:
            fieldfile.save(None, ContentFile(html_output.get_string()),
                    save=False)
            type(book).objects.filter(pk=book.pk).update(**{
                fieldfile.field.attname: fieldfile
            })

            # get ancestor l-tags for adding to new fragments
            ancestor_tags = []
            p = book.parent
            while p:
                ancestor_tags.append(p.book_tag())
                p = p.parent

            # Delete old fragments and create them from scratch
            book.fragments.all().delete()
            # Extract fragments
            closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
            for fragment in closed_fragments.values():
                try:
                    theme_names = [s.strip() for s in fragment.themes.split(',')]
                except AttributeError:
                    continue
                themes = []
                for theme_name in theme_names:
                    if not theme_name:
                        continue
                    tag, created = Tag.objects.get_or_create(
                                        slug=slughifi(theme_name),
                                        category='theme')
                    if created:
                        tag.name = theme_name
                        tag.sort_key = sortify(theme_name.lower())
                        tag.save()
                    themes.append(tag)
                if not themes:
                    continue

                text = fragment.to_string()
                short_text = truncate_html_words(text, 15)
                if text == short_text:
                    short_text = ''
                new_fragment = Fragment.objects.create(anchor=fragment.id, 
                        book=book, text=text, short_text=short_text)

                new_fragment.save()
                new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
            book.html_built.send(sender=book)
            return True
        return False