def save(self, *args, **kwargs):
from catalogue.utils import ExistingFile, remove_zip
try:
old = BookMedia.objects.get(pk=self.pk)
except BookMedia.DoesNotExist:
old = None
else:
# if name changed, change the file name, too
if slughifi(self.name) != slughifi(old.name):
self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
super(BookMedia, self).save(*args, **kwargs)
# remove the zip package for book with modified media
if old:
remove_zip("%s_%s" % (old.book.slug, old.type))
remove_zip("%s_%s" % (self.book.slug, self.type))
extra_info = self.extra_info
if isinstance(extra_info, basestring):
# Walkaround for weird jsonfield 'no-decode' optimization.
extra_info = json.loads(extra_info)
extra_info.update(self.read_meta())
self.extra_info = extra_info
self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
return super(BookMedia, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
from fnpdjango.utils.text.slughifi import slughifi
from catalogue.utils import ExistingFile, remove_zip
try:
old = BookMedia.objects.get(pk=self.pk)
except BookMedia.DoesNotExist:
old = None
else:
# if name changed, change the file name, too
if slughifi(self.name) != slughifi(old.name):
self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
super(BookMedia, self).save(*args, **kwargs)
# remove the zip package for book with modified media
if old:
remove_zip("%s_%s" % (old.book.slug, old.type))
remove_zip("%s_%s" % (self.book.slug, self.type))
extra_info = self.extra_info
if isinstance(extra_info, basestring):
# Walkaround for weird jsonfield 'no-decode' optimization.
extra_info = json.loads(extra_info)
extra_info.update(self.read_meta())
self.extra_info = extra_info
self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
return super(BookMedia, self).save(*args, **kwargs)
def get_dynamic_path(media, filename, ext=None, maxlen=100):
from fnpdjango.utils.text.slughifi import slughifi
# how to put related book's slug here?
if not ext:
# BookMedia case
ext = media.formats[media.type].ext
if media is None or not media.name:
name = slughifi(filename.split(".")[0])
else:
name = slughifi(media.name)
return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
def tags_from_info(info):
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
meta_tags = []
categories = (('kinds', 'kind'), ('genres', 'genre'),
('authors', 'author'), ('epochs', 'epoch'))
for field_name, category in categories:
try:
tag_names = getattr(info, field_name)
except:
try:
tag_names = [getattr(info, category)]
except:
# For instance, Pictures do not have 'genre' field.
continue
for tag_name in tag_names:
tag_sort_key = tag_name
if category == 'author':
tag_sort_key = tag_name.last_name
tag_name = tag_name.readable()
tag, created = Tag.objects.get_or_create(
slug=slughifi(tag_name), category=category)
if created:
tag.name = tag_name
tag.sort_key = sortify(tag_sort_key.lower())
tag.save()
meta_tags.append(tag)
return meta_tags
def tags_from_info(info):
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
meta_tags = []
categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
for field_name, category in categories:
try:
tag_names = getattr(info, field_name)
except:
try:
tag_names = [getattr(info, category)]
except:
# For instance, Pictures do not have 'genre' field.
continue
for tag_name in tag_names:
tag_sort_key = tag_name
if category == 'author':
tag_sort_key = tag_name.last_name
tag_name = tag_name.readable()
tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
if created:
tag.name = tag_name
tag.sort_key = sortify(tag_sort_key.lower())
tag.save()
meta_tags.append(tag)
return meta_tags
def info_args(title, language=None):
""" generate some keywords for comfortable BookInfoCreation """
slug = unicode(slughifi(title))
if language is None:
language = u'pol'
return {
'title': unicode(title),
'url': WLURI.from_slug(slug),
'about': u"http://wolnelektury.pl/example/URI/%s" % slug,
'language': language,
}
def info_args(title, language=None):
""" generate some keywords for comfortable BookInfoCreation """
slug = unicode(slughifi(title))
if language is None:
language = u'pol'
return {
'title': unicode(title),
'url': WLURI.from_slug(slug),
'about': u"http://wolnelektury.pl/example/URI/%s" % slug,
'language': language,
}
def save(self, *args, **kwargs):
from catalogue.utils import ExistingFile, remove_zip
parts_count = BookMedia.objects.filter(book=self.book,
type=self.type).count()
if parts_count == 1:
self.name = self.book.pretty_title()
else:
no = ('%02d' if parts_count < 100 else '%03d') % self.index
self.name = '%s. %s' % (no, self.book.pretty_title())
if self.part_name:
self.name += ', ' + self.part_name
try:
old = BookMedia.objects.get(pk=self.pk)
except BookMedia.DoesNotExist:
old = None
else:
# if name changed, change the file name, too
if slughifi(self.name) != slughifi(old.name):
self.file.save(None,
ExistingFile(self.file.path),
save=False,
leave=True)
super(BookMedia, self).save(*args, **kwargs)
# remove the zip package for book with modified media
if old:
remove_zip("%s_%s" % (old.book.slug, old.type))
remove_zip("%s_%s" % (self.book.slug, self.type))
extra_info = self.extra_info
if isinstance(extra_info, basestring):
# Walkaround for weird jsonfield 'no-decode' optimization.
extra_info = json.loads(extra_info)
extra_info.update(self.read_meta())
self.extra_info = extra_info
self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
return super(BookMedia, self).save(*args, **kwargs)
def gallery(slug, text):
result = {}
m = re.match(META_REGEX, text)
if m:
for line in m.group(1).split('\n'):
try:
k, v = line.split(':', 1)
result[k.strip()] = v.strip()
except ValueError:
continue
gallery = result.get('gallery', slughifi(slug))
if gallery.startswith('/'):
gallery = os.path.basename(gallery)
return gallery
def tags_from_info(info):
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
meta_tags = []
categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
for field_name, category in categories:
try:
tag_names = getattr(info, field_name)
except KeyError:
try:
tag_names = [getattr(info, category)]
except KeyError:
# For instance, Pictures do not have 'genre' field.
continue
for tag_name in tag_names:
lang = getattr(tag_name, 'lang', settings.LANGUAGE_CODE)
tag_sort_key = tag_name
if category == 'author':
tag_sort_key = ' '.join((tag_name.last_name,) + tag_name.first_names)
tag_name = tag_name.readable()
if lang == settings.LANGUAGE_CODE:
# Allow creating new tag, if it's in default language.
tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
if created:
tag_name = unicode(tag_name)
tag.name = tag_name
setattr(tag, "name_%s" % lang, tag_name)
tag.sort_key = sortify(tag_sort_key.lower())
tag.save()
meta_tags.append(tag)
else:
# Ignore unknown tags in non-default languages.
try:
tag = Tag.objects.get(category=category, **{"name_%s" % lang: tag_name})
except Tag.DoesNotExist:
pass
else:
meta_tags.append(tag)
return meta_tags
def tags_from_info(info):
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
meta_tags = []
categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
for field_name, category in categories:
try:
tag_names = getattr(info, field_name)
except:
try:
tag_names = [getattr(info, category)]
except:
# For instance, Pictures do not have 'genre' field.
continue
for tag_name in tag_names:
lang = getattr(tag_name, 'lang', settings.LANGUAGE_CODE)
tag_sort_key = tag_name
if category == 'author':
tag_sort_key = tag_name.last_name
tag_name = tag_name.readable()
if lang == settings.LANGUAGE_CODE:
# Allow creating new tag, if it's in default language.
tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
if created:
tag_name = unicode(tag_name)
tag.name = tag_name
setattr(tag, "name_%s" % lang, tag_name)
tag.sort_key = sortify(tag_sort_key.lower())
tag.save()
meta_tags.append(tag)
else:
# Ignore unknown tags in non-default languages.
try:
tag = Tag.objects.get(category=category, **{"name_%s" % lang: tag_name})
except Tag.DoesNotExist:
pass
else:
meta_tags.append(tag)
return meta_tags
def build(self, fieldfile):
from django.core.files.base import ContentFile
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
from librarian import html
from catalogue.models import Fragment, Tag
book = fieldfile.instance
html_output = self.transform(book.wldocument(parse_dublincore=False),
fieldfile)
# Delete old fragments, create from scratch if necessary.
book.fragments.all().delete()
if html_output:
meta_tags = list(
book.tags.filter(category__in=('author', 'epoch', 'genre',
'kind')))
lang = book.language
lang = LANGUAGES_3TO2.get(lang, lang)
if lang not in [ln[0] for ln in settings.LANGUAGES]:
lang = None
fieldfile.save(None,
ContentFile(html_output.get_string()),
save=False)
type(book).objects.filter(pk=book.pk).update(
**{fieldfile.field.attname: fieldfile})
# Extract fragments
closed_fragments, open_fragments = html.extract_fragments(
fieldfile.path)
for fragment in closed_fragments.values():
try:
theme_names = [
s.strip() for s in fragment.themes.split(',')
]
except AttributeError:
continue
themes = []
for theme_name in theme_names:
if not theme_name:
continue
if lang == settings.LANGUAGE_CODE:
# Allow creating themes if book in default language.
tag, created = Tag.objects.get_or_create(
slug=slughifi(theme_name), category='theme')
if created:
tag.name = theme_name
setattr(tag, "name_%s" % lang, theme_name)
tag.sort_key = sortify(theme_name.lower())
tag.save()
themes.append(tag)
elif lang is not None:
# Don't create unknown themes in non-default languages.
try:
tag = Tag.objects.get(
category='theme',
**{"name_%s" % lang: theme_name})
except Tag.DoesNotExist:
pass
else:
themes.append(tag)
if not themes:
continue
text = fragment.to_string()
short_text = truncate_html_words(text, 15)
if text == short_text:
short_text = ''
new_fragment = Fragment.objects.create(anchor=fragment.id,
book=book,
text=text,
short_text=short_text)
new_fragment.save()
new_fragment.tags = set(meta_tags + themes)
book.html_built.send(sender=type(self), instance=book)
return True
return False
def _file_upload_to(i, _n):
return 'book/%(ext)s/%(name)s.%(ext)s' % {
'ext': i.ext(),
'name': slughifi(i.name)
}
def _file_upload_to(i, _n):
return 'book/%(ext)s/%(name)s.%(ext)s' % {
'ext': i.ext(), 'name': slughifi(i.name)}
def build(self, fieldfile):
from django.core.files.base import ContentFile
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
from librarian import html
from catalogue.models import Fragment, Tag
book = fieldfile.instance
html_output = self.transform(
book.wldocument(parse_dublincore=False),
fieldfile)
# Delete old fragments, create from scratch if necessary.
book.fragments.all().delete()
if html_output:
meta_tags = list(book.tags.filter(
category__in=('author', 'epoch', 'genre', 'kind')))
lang = book.language
lang = LANGUAGES_3TO2.get(lang, lang)
if lang not in [ln[0] for ln in settings.LANGUAGES]:
lang = None
fieldfile.save(None, ContentFile(html_output.get_string()),
save=False)
type(book).objects.filter(pk=book.pk).update(**{
fieldfile.field.attname: fieldfile
})
# Extract fragments
closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
for fragment in closed_fragments.values():
try:
theme_names = [s.strip() for s in fragment.themes.split(',')]
except AttributeError:
continue
themes = []
for theme_name in theme_names:
if not theme_name:
continue
if lang == settings.LANGUAGE_CODE:
# Allow creating themes if book in default language.
tag, created = Tag.objects.get_or_create(
slug=slughifi(theme_name),
category='theme')
if created:
tag.name = theme_name
setattr(tag, "name_%s" % lang, theme_name)
tag.sort_key = sortify(theme_name.lower())
tag.save()
themes.append(tag)
elif lang is not None:
# Don't create unknown themes in non-default languages.
try:
tag = Tag.objects.get(category='theme',
**{"name_%s" % lang: theme_name})
except Tag.DoesNotExist:
pass
else:
themes.append(tag)
if not themes:
continue
text = fragment.to_string()
short_text = truncate_html_words(text, 15)
if text == short_text:
short_text = ''
new_fragment = Fragment.objects.create(anchor=fragment.id,
book=book, text=text, short_text=short_text)
new_fragment.save()
new_fragment.tags = set(meta_tags + themes)
book.html_built.send(sender=type(self), instance=book)
return True
return False
def migrate_file_from_hg(orm, fname, entry):
fname = urlunquote(fname)
print fname
if fname.endswith('.xml'):
fname = fname[:-4]
title = file_to_title(fname)
fname = slughifi(fname)
# create all the needed objects
# what if it already exists?
book = orm.Book.objects.create(
title=title,
slug=fname)
chunk = orm.Chunk.objects.create(
book=book,
number=1,
slug='1')
try:
chunk.stage = orm.ChunkTag.objects.order_by('ordering')[0]
except IndexError:
chunk.stage = None
maxrev = entry.filerev()
gallery_link = None
# this will fail if directory exists
os.makedirs(os.path.join(settings.CATALOGUE_REPO_PATH, str(chunk.pk)))
for rev in xrange(maxrev + 1):
fctx = entry.filectx(rev)
data = fctx.data()
gallery_link = gallery(fname, data)
data = plain_text(data)
# get tags from description
description = fctx.description().decode("utf-8", 'replace')
tags = STAGE_TAGS_RE.findall(description)
tags = [orm.ChunkTag.objects.get(slug=slug.strip()) for slug in tags]
if tags:
max_ordering = max(tags, key=lambda x: x.ordering).ordering
try:
chunk.stage = orm.ChunkTag.objects.filter(ordering__gt=max_ordering).order_by('ordering')[0]
except IndexError:
chunk.stage = None
description = STAGE_TAGS_RE.sub('', description)
author = author_name = author_email = None
author_desc = fctx.user().decode("utf-8", 'replace')
m = AUTHOR_RE.match(author_desc)
if m:
try:
author = orm['auth.User'].objects.get(username=m.group(1), email=m.group(2))
except orm['auth.User'].DoesNotExist:
author_name = m.group(1)
author_email = m.group(2)
else:
author_name = author_desc
head = orm.ChunkChange.objects.create(
tree=chunk,
revision=rev + 1,
created_at=datetime.datetime.fromtimestamp(fctx.date()[0]),
description=description,
author=author,
author_name=author_name,
author_email=author_email,
parent=chunk.head
)
path = "%d/%d" % (chunk.pk, head.pk)
abs_path = os.path.join(settings.CATALOGUE_REPO_PATH, path)
f = open(abs_path, 'wb')
f.write(compress(data))
f.close()
head.data = path
head.tags = tags
head.save()
chunk.head = head
chunk.save()
if gallery_link:
book.gallery = gallery_link
book.save()
def from_xml_file(cls, xml_file, image_file=None, overwrite=False):
"""
Import xml and it's accompanying image file.
If image file is missing, it will be fetched by librarian.picture.ImageStore
which looks for an image file in the same directory the xml is, with extension matching
its mime type.
"""
from sortify import sortify
from django.core.files import File
from librarian.picture import WLPicture, ImageStore
close_xml_file = False
close_image_file = False
# class SimpleImageStore(object):
# def path(self_, slug, mime_type):
# """Returns the image file. Ignores slug ad mime_type."""
# return image_file
if image_file is not None and not isinstance(image_file, File):
image_file = File(open(image_file))
close_image_file = True
if not isinstance(xml_file, File):
xml_file = File(open(xml_file))
close_xml_file = True
try:
# use librarian to parse meta-data
picture_xml = WLPicture.from_file(
xml_file,
image_store=ImageStore(picture_storage.path('images')))
# image_store=SimpleImageStore
picture, created = Picture.objects.get_or_create(
slug=picture_xml.slug)
if not created and not overwrite:
raise Picture.AlreadyExists('Picture %s already exists' %
picture_xml.slug)
picture.title = picture_xml.picture_info.title
motif_tags = set()
for part in picture_xml.partiter():
for motif in part['themes']:
tag, created = catalogue.models.Tag.objects.get_or_create(
slug=slughifi(motif), category='theme')
if created:
tag.name = motif
tag.sort_key = sortify(tag.name)
tag.save()
motif_tags.add(tag)
picture.tags = catalogue.models.Tag.tags_from_info(picture_xml.picture_info) + \
list(motif_tags)
if image_file is not None:
img = image_file
else:
img = picture_xml.image_file()
# FIXME: hardcoded extension
picture.image_file.save(path.basename(picture_xml.image_path),
File(img))
picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
picture.save()
finally:
if close_xml_file:
xml_file.close()
if close_image_file:
image_file.close()
return picture
def from_xml_file(cls, xml_file, image_file=None, overwrite=False):
"""
Import xml and it's accompanying image file.
If image file is missing, it will be fetched by librarian.picture.ImageStore
which looks for an image file in the same directory the xml is, with extension matching
its mime type.
"""
from sortify import sortify
from django.core.files import File
from librarian.picture import WLPicture, ImageStore
close_xml_file = False
close_image_file = False
# class SimpleImageStore(object):
# def path(self_, slug, mime_type):
# """Returns the image file. Ignores slug ad mime_type."""
# return image_file
if image_file is not None and not isinstance(image_file, File):
image_file = File(open(image_file))
close_image_file = True
if not isinstance(xml_file, File):
xml_file = File(open(xml_file))
close_xml_file = True
try:
# use librarian to parse meta-data
picture_xml = WLPicture.from_file(xml_file, image_store=ImageStore(picture_storage.path("images")))
# image_store=SimpleImageStore
picture, created = Picture.objects.get_or_create(slug=picture_xml.slug)
if not created and not overwrite:
raise Picture.AlreadyExists("Picture %s already exists" % picture_xml.slug)
picture.title = picture_xml.picture_info.title
motif_tags = set()
for part in picture_xml.partiter():
for motif in part["themes"]:
tag, created = catalogue.models.Tag.objects.get_or_create(slug=slughifi(motif), category="theme")
if created:
tag.name = motif
tag.sort_key = sortify(tag.name)
tag.save()
motif_tags.add(tag)
picture.tags = catalogue.models.Tag.tags_from_info(picture_xml.picture_info) + list(motif_tags)
if image_file is not None:
img = image_file
else:
img = picture_xml.image_file()
# FIXME: hardcoded extension
picture.image_file.save(path.basename(picture_xml.image_path), File(img))
picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
picture.save()
finally:
if close_xml_file:
xml_file.close()
if close_image_file:
image_file.close()
return picture
def handle(self, *redmine_csv, **options):
self.style = color_style()
redakcja = options.get('redakcja')
verbose = options.get('verbose')
force = options.get('force')
if not redmine_csv:
if verbose:
print "Using default Redmine CSV URL:", REDMINE_CSV
redmine_csv = REDMINE_CSV
# Start transaction management.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
redakcja_link = re.compile(re.escape(redakcja) + r'([-_.:?&%/a-zA-Z0-9]*)')
all_tickets = 0
all_chunks = 0
done_tickets = 0
done_chunks = 0
empty_users = 0
unknown_users = {}
unknown_books = []
forced = []
if verbose:
print 'Downloading CSV file'
for r in csv.reader(urllib2.urlopen(redmine_csv)):
if r[0] == '#':
continue
all_tickets += 1
username = r[6]
if not username:
if verbose:
print "Empty user, skipping"
empty_users += 1
continue
first_name, last_name = unicode(username, 'utf-8').rsplit(u' ', 1)
try:
user = User.objects.get(first_name=first_name, last_name=last_name)
except User.DoesNotExist:
print self.style.ERROR('Unknown user: '******'utf-8', 'ignore')
if fname.endswith('.xml'):
fname = fname[:-4]
fname = fname.replace(' ', '_')
fname = slughifi(fname)
chunks = Chunk.objects.filter(book__slug=fname)
if not chunks:
print self.style.ERROR('Unknown book: ' + fname)
unknown_books.append(fname)
continue
all_chunks += chunks.count()
for chunk in chunks:
if chunk.user:
if chunk.user == user:
continue
else:
forced.append((chunk, chunk.user, user))
if force:
print self.style.WARNING(
'%s assigned to %s, forcing change to %s.' %
(chunk.pretty_name(), chunk.user, user))
else:
print self.style.WARNING(
'%s assigned to %s not to %s, skipping.' %
(chunk.pretty_name(), chunk.user, user))
continue
chunk.user = user
chunk.save()
ticket_done = True
done_chunks += 1
if ticket_done:
done_tickets += 1
# Print results
print
print "Results:"
print "Assignments imported from %d/%d tickets to %d/%d relevalt chunks." % (
done_tickets, all_tickets, done_chunks, all_chunks)
if empty_users:
print "%d tickets were unassigned." % empty_users
if forced:
print "%d assignments conficts (%s):" % (
len(forced), "changed" if force else "left")
for chunk, orig, user in forced:
print " %s: \t%s \t-> %s" % (
chunk.pretty_name(), orig.username, user.username)
if unknown_books:
print "%d unknown books:" % len(unknown_books)
for fname in unknown_books:
print " %s" % fname
if unknown_users:
print "%d unknown users:" % len(unknown_users)
for name in unknown_users:
print " %s (%d tickets)" % (name, unknown_users[name])
print
transaction.commit()
transaction.leave_transaction_management()
def build(self, fieldfile):
from django.core.files.base import ContentFile
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
from librarian import html
from catalogue.models import Fragment, Tag
book = fieldfile.instance
meta_tags = list(
book.tags.filter(category__in=('author', 'epoch', 'genre',
'kind')))
book_tag = book.book_tag()
html_output = self.transform(book.wldocument(parse_dublincore=False),
fieldfile)
if html_output:
fieldfile.save(None,
ContentFile(html_output.get_string()),
save=False)
type(book).objects.filter(pk=book.pk).update(
**{fieldfile.field.attname: fieldfile})
# get ancestor l-tags for adding to new fragments
ancestor_tags = []
p = book.parent
while p:
ancestor_tags.append(p.book_tag())
p = p.parent
# Delete old fragments and create them from scratch
book.fragments.all().delete()
# Extract fragments
closed_fragments, open_fragments = html.extract_fragments(
fieldfile.path)
for fragment in closed_fragments.values():
try:
theme_names = [
s.strip() for s in fragment.themes.split(',')
]
except AttributeError:
continue
themes = []
for theme_name in theme_names:
if not theme_name:
continue
tag, created = Tag.objects.get_or_create(
slug=slughifi(theme_name), category='theme')
if created:
tag.name = theme_name
tag.sort_key = sortify(theme_name.lower())
tag.save()
themes.append(tag)
if not themes:
continue
text = fragment.to_string()
short_text = truncate_html_words(text, 15)
if text == short_text:
short_text = ''
new_fragment = Fragment.objects.create(anchor=fragment.id,
book=book,
text=text,
short_text=short_text)
new_fragment.save()
new_fragment.tags = set(meta_tags + themes + [book_tag] +
ancestor_tags)
book.html_built.send(sender=book)
return True
return False
def from_xml_file(cls, xml_file, image_file=None, image_store=None, overwrite=False):
"""
Import xml and it's accompanying image file.
If image file is missing, it will be fetched by librarian.picture.ImageStore
which looks for an image file in the same directory the xml is, with extension matching
its mime type.
"""
from sortify import sortify
from django.core.files import File
from librarian.picture import WLPicture, ImageStore
close_xml_file = False
close_image_file = False
if image_file is not None and not isinstance(image_file, File):
image_file = File(open(image_file))
close_image_file = True
if not isinstance(xml_file, File):
xml_file = File(open(xml_file))
close_xml_file = True
with transaction.atomic():
# use librarian to parse meta-data
if image_store is None:
image_store = ImageStore(picture_storage.path('images'))
picture_xml = WLPicture.from_file(xml_file, image_store=image_store)
picture, created = Picture.objects.get_or_create(slug=picture_xml.slug[:120])
if not created and not overwrite:
raise Picture.AlreadyExists('Picture %s already exists' % picture_xml.slug)
picture.areas.all().delete()
picture.title = unicode(picture_xml.picture_info.title)
picture.extra_info = picture_xml.picture_info.to_dict()
picture_tags = set(catalogue.models.Tag.tags_from_info(picture_xml.picture_info))
motif_tags = set()
thing_tags = set()
area_data = {'themes': {}, 'things': {}}
# Treat all names in picture XML as in default language.
lang = settings.LANGUAGE_CODE
for part in picture_xml.partiter():
if picture_xml.frame:
c = picture_xml.frame[0]
part['coords'] = [[p[0] - c[0], p[1] - c[1]] for p in part['coords']]
if part.get('object', None) is not None:
_tags = set()
for objname in part['object'].split(','):
objname = objname.strip().capitalize()
tag, created = catalogue.models.Tag.objects.get_or_create(
slug=slughifi(objname), category='thing')
if created:
tag.name = objname
setattr(tag, 'name_%s' % lang, tag.name)
tag.sort_key = sortify(tag.name)
tag.save()
# thing_tags.add(tag)
area_data['things'][tag.slug] = {
'object': objname,
'coords': part['coords'],
}
_tags.add(tag)
area = PictureArea.rectangle(picture, 'thing', part['coords'])
area.save()
area.tags = _tags
else:
_tags = set()
for motifs in part['themes']:
for motif in motifs.split(','):
tag, created = catalogue.models.Tag.objects.get_or_create(
slug=slughifi(motif), category='theme')
if created:
tag.name = motif
tag.sort_key = sortify(tag.name)
tag.save()
# motif_tags.add(tag)
_tags.add(tag)
area_data['themes'][tag.slug] = {
'theme': motif,
'coords': part['coords']
}
logging.debug("coords for theme: %s" % part['coords'])
area = PictureArea.rectangle(picture, 'theme', part['coords'])
area.save()
area.tags = _tags.union(picture_tags)
picture.tags = picture_tags.union(motif_tags).union(thing_tags)
picture.areas_json = area_data
if image_file is not None:
img = image_file
else:
img = picture_xml.image_file()
modified = cls.crop_to_frame(picture_xml, img)
modified = cls.add_source_note(picture_xml, modified)
picture.width, picture.height = modified.size
modified_file = StringIO()
modified.save(modified_file, format='JPEG', quality=95)
# FIXME: hardcoded extension - detect from DC format or orginal filename
picture.image_file.save(path.basename(picture_xml.image_path), File(modified_file))
picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
picture.save()
tasks.generate_picture_html(picture.id)
if close_xml_file:
xml_file.close()
if close_image_file:
image_file.close()
return picture
def upload(request):
if request.method == "POST":
form = forms.DocumentsUploadForm(request.POST, request.FILES)
if form.is_valid():
from fnpdjango.utils.text.slughifi import slughifi
if request.user.is_authenticated():
creator = request.user
else:
creator = None
zip = form.cleaned_data['zip']
skipped_list = []
ok_list = []
error_list = []
slugs = {}
existing = [book.slug for book in Book.objects.all()]
for filename in zip.namelist():
if filename[-1] == '/':
continue
title = os.path.basename(filename)[:-4]
slug = slughifi(title)
if not (slug and filename.endswith('.xml')):
skipped_list.append(filename)
elif slug in slugs:
error_list.append((filename, slug, _('Slug already used for %s' % slugs[slug])))
elif slug in existing:
error_list.append((filename, slug, _('Slug already used in repository.')))
else:
try:
zip.read(filename).decode('utf-8') # test read
ok_list.append((filename, slug, title))
except UnicodeDecodeError:
error_list.append((filename, title, _('File should be UTF-8 encoded.')))
slugs[slug] = filename
if not error_list:
for filename, slug, title in ok_list:
book = Book.create(
text=zip.read(filename).decode('utf-8'),
creator=creator,
slug=slug,
title=title,
)
return render(request, "catalogue/document_upload.html", {
"form": form,
"ok_list": ok_list,
"skipped_list": skipped_list,
"error_list": error_list,
"logout_to": '/',
})
else:
form = forms.DocumentsUploadForm()
return render(request, "catalogue/document_upload.html", {
"form": form,
"logout_to": '/',
})
def build(self, fieldfile):
from django.core.files.base import ContentFile
from fnpdjango.utils.text.slughifi import slughifi
from sortify import sortify
from librarian import html
from catalogue.models import Fragment, Tag
book = fieldfile.instance
meta_tags = list(book.tags.filter(
category__in=('author', 'epoch', 'genre', 'kind')))
book_tag = book.book_tag()
html_output = self.transform(
book.wldocument(parse_dublincore=False),
fieldfile)
if html_output:
fieldfile.save(None, ContentFile(html_output.get_string()),
save=False)
type(book).objects.filter(pk=book.pk).update(**{
fieldfile.field.attname: fieldfile
})
# get ancestor l-tags for adding to new fragments
ancestor_tags = []
p = book.parent
while p:
ancestor_tags.append(p.book_tag())
p = p.parent
# Delete old fragments and create them from scratch
book.fragments.all().delete()
# Extract fragments
closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
for fragment in closed_fragments.values():
try:
theme_names = [s.strip() for s in fragment.themes.split(',')]
except AttributeError:
continue
themes = []
for theme_name in theme_names:
if not theme_name:
continue
tag, created = Tag.objects.get_or_create(
slug=slughifi(theme_name),
category='theme')
if created:
tag.name = theme_name
tag.sort_key = sortify(theme_name.lower())
tag.save()
themes.append(tag)
if not themes:
continue
text = fragment.to_string()
short_text = truncate_html_words(text, 15)
if text == short_text:
short_text = ''
new_fragment = Fragment.objects.create(anchor=fragment.id,
book=book, text=text, short_text=short_text)
new_fragment.save()
new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
book.html_built.send(sender=book)
return True
return False