def test_author_get_display_name(self):
"""
Test that the display name of an author is correct
"""
# GIVEN: An author
author = Author()
author.display_name = "John Doe"
# WHEN: We call the get_display_name() function
display_name = author.get_display_name()
# THEN: It should return only the name
self.assertEqual("John Doe", display_name)
def test_author_get_display_name_with_type_translation(self):
"""
Test that the display name of an author with a type is correct (Translation)
"""
# GIVEN: An author
author = Author()
author.display_name = "John Doe"
# WHEN: We call the get_display_name() function
display_name = author.get_display_name(AuthorType.Translation)
# THEN: It should return the name with the type in brackets
self.assertEqual("John Doe (Translation)", display_name)
def test_author_get_display_name_with_type_translation(self):
"""
Test that the display name of an author with a type is correct (Translation)
"""
# GIVEN: An author
author = Author()
author.display_name = "John Doe"
# WHEN: We call the get_display_name() function
display_name = author.get_display_name(AuthorType.Translation)
# THEN: It should return the name with the type in brackets
self.assertEqual("John Doe (Translation)", display_name)
def test_author_get_display_name_with_type_words(self):
"""
Test that the display name of an author with a type is correct (Words)
"""
# GIVEN: An author
author = Author()
author.display_name = "John Doe"
# WHEN: We call the get_display_name() function
display_name = author.get_display_name(AuthorType.Words)
# THEN: It should return the name with the type in brackets
assert "John Doe (Words)" == display_name
def test_author_get_display_name(self):
"""
Test that the display name of an author is correct
"""
# GIVEN: An author
author = Author()
author.display_name = "John Doe"
# WHEN: We call the get_display_name() function
display_name = author.get_display_name()
# THEN: It should return only the name
self.assertEqual("John Doe", display_name)
def _process_authors(self, properties, song):
"""
Adds the authors specified in the XML to the song.
:param properties: The property object (lxml.objectify.ObjectifiedElement).
:param song: The song object
"""
authors = []
if hasattr(properties, 'authors'):
for author in properties.authors.author:
display_name = self._text(author)
author_type = author.get('type', '')
# As of 0.8 OpenLyrics supports these 3 author types
if author_type not in ('words', 'music', 'translation'):
author_type = ''
if display_name:
# Check if an author is listed for both music and words. In that case we use a special type
if author_type == 'words' and (display_name, 'music') in authors:
authors.remove((display_name, 'music'))
authors.append((display_name, 'words+music'))
elif author_type == 'music' and (display_name, 'words') in authors:
authors.remove((display_name, 'words'))
authors.append((display_name, 'words+music'))
else:
authors.append((display_name, author_type))
for (display_name, author_type) in authors:
author = self.manager.get_object_filtered(Author, Author.display_name == display_name)
if author is None:
# We need to create a new author, as the author does not exist.
author = Author.populate(display_name=display_name,
last_name=display_name.split(' ')[-1],
first_name=' '.join(display_name.split(' ')[:-1]))
song.add_author(author, author_type)
def clean_song(manager, song):
"""
Cleans the search title, rebuilds the search lyrics, adds a default author if the song does not have one and other
clean ups. This should always called when a new song is added or changed.
:param manager: The song database manager object.
:param song: The song object.
"""
from .openlyricsxml import SongXML
if song.title:
song.title = clean_title(song.title)
else:
song.title = ''
if song.alternate_title:
song.alternate_title = clean_title(song.alternate_title)
else:
song.alternate_title = ''
song.search_title = clean_string(song.title) + '@' + clean_string(song.alternate_title)
if isinstance(song.lyrics, bytes):
song.lyrics = str(song.lyrics, encoding='utf8')
verses = SongXML().get_verses(song.lyrics)
song.search_lyrics = ' '.join([clean_string(clean_tags(verse[1])) for verse in verses])
# The song does not have any author, add one.
if not song.authors_songs:
name = SongStrings.AuthorUnknown
author = manager.get_object_filtered(Author, Author.display_name == name)
if author is None:
author = Author.populate(display_name=name, last_name='', first_name='')
song.add_author(author)
if song.copyright:
song.copyright = CONTROL_CHARS.sub('', song.copyright).strip()
def _process_authors(self, properties, song):
"""
Adds the authors specified in the XML to the song.
``properties``
The property object (lxml.objectify.ObjectifiedElement).
``song``
The song object.
"""
authors = []
if hasattr(properties, u'authors'):
for author in properties.authors.author:
display_name = self._text(author)
if display_name:
authors.append(display_name)
for display_name in authors:
author = self.manager.get_object_filtered(Author,
Author.display_name == display_name)
if author is None:
# We need to create a new author, as the author does not exist.
author = Author.populate(display_name=display_name,
last_name=display_name.split(u' ')[-1],
first_name=u' '.join(display_name.split(u' ')[:-1]))
song.authors.append(author)
def _process_authors(self, properties, song):
"""
Adds the authors specified in the XML to the song.
:param properties: The property object (lxml.objectify.ObjectifiedElement).
:param song: The song object
"""
authors = []
if hasattr(properties, 'authors'):
for author in properties.authors.author:
display_name = self._text(author)
author_type = author.get('type', '')
# As of 0.8 OpenLyrics supports these 3 author types
if author_type not in ('words', 'music', 'translation'):
author_type = ''
if display_name:
# Check if an author is listed for both music and words. In that case we use a special type
if author_type == 'words' and (display_name, 'music') in authors:
authors.remove((display_name, 'music'))
authors.append((display_name, 'words+music'))
elif author_type == 'music' and (display_name, 'words') in authors:
authors.remove((display_name, 'words'))
authors.append((display_name, 'words+music'))
else:
authors.append((display_name, author_type))
for (display_name, author_type) in authors:
author = self.manager.get_object_filtered(Author, Author.display_name == display_name)
if author is None:
# We need to create a new author, as the author does not exist.
author = Author.populate(display_name=display_name,
last_name=display_name.split(' ')[-1],
first_name=' '.join(display_name.split(' ')[:-1]))
song.add_author(author, author_type)
def clean_song(manager, song):
"""
Cleans the search title, rebuilds the search lyrics, adds a default author if the song does not have one and other
clean ups. This should always called when a new song is added or changed.
:param manager: The song database manager object.
:param song: The song object.
"""
from .openlyricsxml import SongXML
if song.title:
song.title = clean_title(song.title)
else:
song.title = ''
if song.alternate_title:
song.alternate_title = clean_title(song.alternate_title)
else:
song.alternate_title = ''
song.search_title = clean_string(song.title) + '@' + clean_string(song.alternate_title)
if isinstance(song.lyrics, bytes):
song.lyrics = str(song.lyrics, encoding='utf8')
verses = SongXML().get_verses(song.lyrics)
song.search_lyrics = ' '.join([clean_string(verse[1]) for verse in verses])
# The song does not have any author, add one.
if not song.authors_songs:
name = SongStrings.AuthorUnknown
author = manager.get_object_filtered(Author, Author.display_name == name)
if author is None:
author = Author.populate(display_name=name, last_name='', first_name='')
song.add_author(author)
if song.copyright:
song.copyright = CONTROL_CHARS.sub('', song.copyright).strip()
def test_add_author_with_type(self):
"""
Test adding an author with a type specified to a song
"""
# GIVEN: A song and an author
song = Song()
song.authors_songs = []
author = Author()
author.first_name = "Max"
author.last_name = "Mustermann"
# WHEN: We add an author to the song
song.add_author(author, AuthorType.Words)
# THEN: The author should have been added with author_type=None
self.assertEqual(1, len(song.authors_songs))
self.assertEqual("Max", song.authors_songs[0].author.first_name)
self.assertEqual("Mustermann", song.authors_songs[0].author.last_name)
self.assertEqual(AuthorType.Words, song.authors_songs[0].author_type)
def test_add_author_with_type(self):
"""
Test adding an author with a type specified to a song
"""
# GIVEN: A song and an author
song = Song()
song.authors_songs = []
author = Author()
author.first_name = "Max"
author.last_name = "Mustermann"
# WHEN: We add an author to the song
song.add_author(author, AuthorType.Words)
# THEN: The author should have been added with author_type=None
self.assertEqual(1, len(song.authors_songs))
self.assertEqual("Max", song.authors_songs[0].author.first_name)
self.assertEqual("Mustermann", song.authors_songs[0].author.last_name)
self.assertEqual(AuthorType.Words, song.authors_songs[0].author_type)
def test_add_author(self):
"""
Test adding an author to a song
"""
# GIVEN: A song and an author
song = Song()
song.authors_songs = []
author = Author()
author.first_name = "Max"
author.last_name = "Mustermann"
# WHEN: We add an author to the song
song.add_author(author)
# THEN: The author should have been added with author_type=None
assert 1 == len(song.authors_songs)
assert "Max" == song.authors_songs[0].author.first_name
assert "Mustermann" == song.authors_songs[0].author.last_name
assert song.authors_songs[0].author_type is None
def on_author_add_button_clicked(self):
"""
Add the author to the list of authors associated with this song when the button is clicked.
"""
item = int(self.authors_combo_box.currentIndex())
text = self.authors_combo_box.currentText().strip(' \r\n\t')
# This if statement is for OS X, which doesn't seem to work well with
# the QCompleter autocompletion class. See bug #812628.
if text in self.authors:
# Index 0 is a blank string, so add 1
item = self.authors.index(text) + 1
if item == 0 and text:
if QtGui.QMessageBox.question(self,
translate('SongsPlugin.EditSongForm', 'Add Author'),
translate('SongsPlugin.EditSongForm', 'This author does not exist, do you want to add them?'),
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, QtGui.QMessageBox.Yes) == QtGui.QMessageBox.Yes:
if text.find(' ') == -1:
author = Author.populate(first_name='', last_name='', display_name=text)
else:
author = Author.populate(first_name=text.rsplit(' ', 1)[0],
last_name=text.rsplit(' ', 1)[1], display_name=text)
self.manager.save_object(author)
self._add_author_to_list(author)
self.load_authors()
self.authors_combo_box.setCurrentIndex(0)
else:
return
elif item > 0:
item_id = (self.authors_combo_box.itemData(item))
author = self.manager.get_object(Author, item_id)
if self.authors_list_view.findItems(str(author.display_name),
QtCore.Qt.MatchExactly):
critical_error_message_box(
message=translate('SongsPlugin.EditSongForm', 'This author is already in the list.'))
else:
self._add_author_to_list(author)
self.authors_combo_box.setCurrentIndex(0)
else:
QtGui.QMessageBox.warning(self, UiStrings().NISs,
translate('SongsPlugin.EditSongForm', 'You have not selected a valid author. Either select an author '
'from the list, or type in a new author and click the "Add Author to Song" button to add '
'the new author.'))
def save_song(self, song):
"""
Save a song to the database, using the db_manager
:param song:
:return:
"""
db_song = Song.populate(title=song['title'],
copyright=song['copyright'],
ccli_number=song['ccli_number'])
song_xml = SongXML()
verse_order = []
for verse in song['verses']:
if ' ' in verse['label']:
verse_type, verse_number = verse['label'].split(' ', 1)
else:
verse_type = verse['label']
verse_number = 1
verse_type = VerseType.from_loose_input(verse_type)
verse_number = int(verse_number)
song_xml.add_verse_to_lyrics(VerseType.tags[verse_type],
verse_number, verse['lyrics'])
verse_order.append('{tag}{number}'.format(
tag=VerseType.tags[verse_type], number=verse_number))
db_song.verse_order = ' '.join(verse_order)
db_song.lyrics = song_xml.extract_xml()
clean_song(self.db_manager, db_song)
self.db_manager.save_object(db_song)
db_song.authors_songs = []
for author_name in song['authors']:
author = self.db_manager.get_object_filtered(
Author, Author.display_name == author_name)
if not author:
name_parts = author_name.rsplit(' ', 1)
first_name = name_parts[0]
if len(name_parts) == 1:
last_name = ''
else:
last_name = name_parts[1]
author = Author.populate(first_name=first_name,
last_name=last_name,
display_name=author_name)
db_song.add_author(author)
for topic_name in song.get('topics', []):
topic = self.db_manager.get_object_filtered(
Topic, Topic.name == topic_name)
if not topic:
topic = Topic.populate(name=topic_name)
db_song.topics.append(topic)
self.db_manager.save_object(db_song)
return db_song
def test_remove_author(self):
"""
Test removing an author from a song
"""
# GIVEN: A song with an author
song = Song()
song.authors_songs = []
author = Author()
song.add_author(author)
# WHEN: We remove the author
song.remove_author(author)
# THEN: It should have been removed
self.assertEqual(0, len(song.authors_songs))
def test_remove_author_with_type(self):
"""
Test removing an author with a type specified from a song
"""
# GIVEN: A song with two authors
song = Song()
song.authors_songs = []
author = Author()
song.add_author(author)
song.add_author(author, AuthorType.Translation)
# WHEN: We remove the author with a certain type
song.remove_author(author, AuthorType.Translation)
# THEN: It should have been removed and the other author should still be there
self.assertEqual(1, len(song.authors_songs))
self.assertEqual(None, song.authors_songs[0].author_type)
def on_add_author_button_clicked(self):
"""
Add an author to the list.
"""
self.author_form.auto_display_name = True
if self.author_form.exec():
author = Author.populate(
first_name=self.author_form.first_name,
last_name=self.author_form.last_name,
display_name=self.author_form.display_name
)
if self.check_author_exists(author):
if self.manager.save_object(author):
self.reset_authors()
else:
critical_error_message_box(
message=translate('SongsPlugin.SongMaintenanceForm', 'Could not add your author.'))
else:
critical_error_message_box(
message=translate('SongsPlugin.SongMaintenanceForm', 'This author already exists.'))
def on_add_author_button_clicked(self):
"""
Add an author to the list.
"""
self.author_form.auto_display_name = True
if self.author_form.exec_():
author = Author.populate(
first_name=self.author_form.first_name,
last_name=self.author_form.last_name,
display_name=self.author_form.display_name
)
if self.check_author_exists(author):
if self.manager.save_object(author):
self.reset_authors()
else:
critical_error_message_box(
message=translate('SongsPlugin.SongMaintenanceForm', 'Could not add your author.'))
else:
critical_error_message_box(
message=translate('SongsPlugin.SongMaintenanceForm', 'This author already exists.'))
def do_import(self, progress_dialog=None):
"""
Run the import for an OpenLP version 2 song database.
:param progress_dialog: The QProgressDialog used when importing songs from the FRW.
"""
class OldAuthorSong(BaseModel):
"""
Maps to the authors_songs table
"""
pass
class OldAuthor(BaseModel):
"""
Maps to the authors table
"""
pass
class OldBook(BaseModel):
"""
Maps to the songbooks table
"""
pass
class OldMediaFile(BaseModel):
"""
Maps to the media_files table
"""
pass
class OldSong(BaseModel):
"""
Maps to the songs table
"""
pass
class OldTopic(BaseModel):
"""
Maps to the topics table
"""
pass
class OldSongBookEntry(BaseModel):
"""
Maps to the songs_songbooks table
"""
pass
# Check the file type
if self.import_source.suffix != '.sqlite':
self.log_error(
self.import_source,
translate('SongsPlugin.OpenLPSongImport',
'Not a valid OpenLP 2 song database.'))
return
self.import_source = 'sqlite:///{url}'.format(url=self.import_source)
# Load the db file and reflect it
engine = create_engine(self.import_source)
source_meta = MetaData()
source_meta.reflect(engine)
self.source_session = scoped_session(sessionmaker(bind=engine))
# Run some checks to see which version of the database we have
table_list = list(source_meta.tables.keys())
if 'media_files' in table_list:
has_media_files = True
else:
has_media_files = False
if 'songs_songbooks' in table_list:
has_songs_books = True
else:
has_songs_books = False
if 'authors_songs' in table_list:
has_authors_songs = True
else:
has_authors_songs = False
# Load up the tabls and map them out
try:
source_authors_table = source_meta.tables['authors']
source_song_books_table = source_meta.tables['song_books']
source_songs_table = source_meta.tables['songs']
source_topics_table = source_meta.tables['topics']
source_authors_songs_table = source_meta.tables['authors_songs']
source_songs_topics_table = source_meta.tables['songs_topics']
source_media_files_songs_table = None
except KeyError:
self.log_error(
self.import_source,
translate('SongsPlugin.OpenLPSongImport',
'Not a valid OpenLP 2 song database.'))
return
# Set up media_files relations
if has_media_files:
source_media_files_table = source_meta.tables['media_files']
source_media_files_songs_table = source_meta.tables.get(
'media_files_songs')
try:
class_mapper(OldMediaFile)
except UnmappedClassError:
mapper(OldMediaFile, source_media_files_table)
if has_songs_books:
source_songs_songbooks_table = source_meta.tables[
'songs_songbooks']
try:
class_mapper(OldSongBookEntry)
except UnmappedClassError:
mapper(OldSongBookEntry,
source_songs_songbooks_table,
properties={'songbook': relation(OldBook)})
if has_authors_songs:
try:
class_mapper(OldAuthorSong)
except UnmappedClassError:
mapper(OldAuthorSong, source_authors_songs_table)
if has_authors_songs and 'author_type' in source_authors_songs_table.c.keys(
):
has_author_type = True
else:
has_author_type = False
# Set up the songs relationships
song_props = {
'authors':
relation(OldAuthor,
backref='songs',
secondary=source_authors_songs_table),
'topics':
relation(OldTopic,
backref='songs',
secondary=source_songs_topics_table)
}
if has_media_files:
if isinstance(source_media_files_songs_table, Table):
song_props['media_files'] = relation(
OldMediaFile,
backref='songs',
secondary=source_media_files_songs_table)
else:
song_props['media_files'] = \
relation(OldMediaFile, backref='songs',
foreign_keys=[source_media_files_table.c.song_id],
primaryjoin=source_songs_table.c.id == source_media_files_table.c.song_id)
if has_songs_books:
song_props['songbook_entries'] = relation(
OldSongBookEntry, backref='song', cascade='all, delete-orphan')
else:
song_props['book'] = relation(OldBook, backref='songs')
if has_authors_songs:
song_props['authors_songs'] = relation(OldAuthorSong)
# Map the rest of the tables
try:
class_mapper(OldAuthor)
except UnmappedClassError:
mapper(OldAuthor, source_authors_table)
try:
class_mapper(OldBook)
except UnmappedClassError:
mapper(OldBook, source_song_books_table)
try:
class_mapper(OldSong)
except UnmappedClassError:
mapper(OldSong, source_songs_table, properties=song_props)
try:
class_mapper(OldTopic)
except UnmappedClassError:
mapper(OldTopic, source_topics_table)
source_songs = self.source_session.query(OldSong).all()
if self.import_wizard:
self.import_wizard.progress_bar.setMaximum(len(source_songs))
for song in source_songs:
new_song = Song()
new_song.title = song.title
if has_media_files and hasattr(song, 'alternate_title'):
new_song.alternate_title = song.alternate_title
else:
old_titles = song.search_title.split('@')
if len(old_titles) > 1:
new_song.alternate_title = old_titles[1]
# Transfer the values to the new song object
new_song.search_title = ''
new_song.search_lyrics = ''
new_song.lyrics = song.lyrics
new_song.verse_order = song.verse_order
new_song.copyright = song.copyright
new_song.comments = song.comments
new_song.theme_name = song.theme_name
new_song.ccli_number = song.ccli_number
if hasattr(song, 'song_number') and song.song_number:
new_song.song_number = song.song_number
# Find or create all the authors and add them to the new song object
for author in song.authors:
existing_author = self.manager.get_object_filtered(
Author, Author.display_name == author.display_name)
if not existing_author:
existing_author = Author.populate(
first_name=author.first_name,
last_name=author.last_name,
display_name=author.display_name)
# If this is a new database, we need to import the author_type too
author_type = None
if has_author_type:
for author_song in song.authors_songs:
if author_song.author_id == author.id:
author_type = author_song.author_type
break
new_song.add_author(existing_author, author_type)
# Find or create all the topics and add them to the new song object
if song.topics:
for topic in song.topics:
existing_topic = self.manager.get_object_filtered(
Topic, Topic.name == topic.name)
if not existing_topic:
existing_topic = Topic.populate(name=topic.name)
new_song.topics.append(existing_topic)
# Find or create all the songbooks and add them to the new song object
if has_songs_books and song.songbook_entries:
for entry in song.songbook_entries:
existing_book = self.manager.get_object_filtered(
Book, Book.name == entry.songbook.name)
if not existing_book:
existing_book = Book.populate(
name=entry.songbook.name,
publisher=entry.songbook.publisher)
new_song.add_songbook_entry(existing_book, entry.entry)
elif hasattr(song, 'book') and song.book:
existing_book = self.manager.get_object_filtered(
Book, Book.name == song.book.name)
if not existing_book:
existing_book = Book.populate(
name=song.book.name, publisher=song.book.publisher)
# Get the song_number from "songs" table "song_number" field. (This is db structure from 2.2.1)
# If there's a number, add it to the song, otherwise it will be "".
existing_number = song.song_number if hasattr(
song, 'song_number') else ''
if existing_number:
new_song.add_songbook_entry(existing_book, existing_number)
else:
new_song.add_songbook_entry(existing_book, '')
# Find or create all the media files and add them to the new song object
if has_media_files and song.media_files:
for media_file in song.media_files:
existing_media_file = self.manager.get_object_filtered(
MediaFile, MediaFile.file_path == media_file.file_path)
if existing_media_file:
new_song.media_files.append(existing_media_file)
else:
new_song.media_files.append(
MediaFile.populate(file_name=media_file.file_name))
clean_song(self.manager, new_song)
self.manager.save_object(new_song)
if progress_dialog:
progress_dialog.setValue(progress_dialog.value() + 1)
progress_dialog.setLabelText(
WizardStrings.ImportingType.format(source=new_song.title))
else:
self.import_wizard.increment_progress_bar(
WizardStrings.ImportingType.format(source=new_song.title))
if self.stop_import_flag:
break
self.source_session.close()
engine.dispose()
def do_import(self, progress_dialog=None):
"""
Run the import for an OpenLP version 2 song database.
:param progress_dialog: The QProgressDialog used when importing songs from the FRW.
"""
class OldAuthor(BaseModel):
"""
Author model
"""
pass
class OldBook(BaseModel):
"""
Book model
"""
pass
class OldMediaFile(BaseModel):
"""
MediaFile model
"""
pass
class OldSong(BaseModel):
"""
Song model
"""
pass
class OldTopic(BaseModel):
"""
Topic model
"""
pass
# Check the file type
if not self.import_source.endswith('.sqlite'):
self.log_error(self.import_source, translate('SongsPlugin.OpenLPSongImport',
'Not a valid OpenLP 2.0 song database.'))
return
self.import_source = 'sqlite:///%s' % self.import_source
# Load the db file
engine = create_engine(self.import_source)
source_meta = MetaData()
source_meta.reflect(engine)
self.source_session = scoped_session(sessionmaker(bind=engine))
if 'media_files' in list(source_meta.tables.keys()):
has_media_files = True
else:
has_media_files = False
source_authors_table = source_meta.tables['authors']
source_song_books_table = source_meta.tables['song_books']
source_songs_table = source_meta.tables['songs']
source_topics_table = source_meta.tables['topics']
source_authors_songs_table = source_meta.tables['authors_songs']
source_songs_topics_table = source_meta.tables['songs_topics']
source_media_files_songs_table = None
if has_media_files:
source_media_files_table = source_meta.tables['media_files']
source_media_files_songs_table = source_meta.tables.get('media_files_songs')
try:
class_mapper(OldMediaFile)
except UnmappedClassError:
mapper(OldMediaFile, source_media_files_table)
song_props = {
'authors': relation(OldAuthor, backref='songs', secondary=source_authors_songs_table),
'book': relation(OldBook, backref='songs'),
'topics': relation(OldTopic, backref='songs', secondary=source_songs_topics_table)
}
if has_media_files:
if isinstance(source_media_files_songs_table, Table):
song_props['media_files'] = relation(OldMediaFile, backref='songs',
secondary=source_media_files_songs_table)
else:
song_props['media_files'] = \
relation(OldMediaFile, backref='songs',
foreign_keys=[source_media_files_table.c.song_id],
primaryjoin=source_songs_table.c.id == source_media_files_table.c.song_id)
try:
class_mapper(OldAuthor)
except UnmappedClassError:
mapper(OldAuthor, source_authors_table)
try:
class_mapper(OldBook)
except UnmappedClassError:
mapper(OldBook, source_song_books_table)
try:
class_mapper(OldSong)
except UnmappedClassError:
mapper(OldSong, source_songs_table, properties=song_props)
try:
class_mapper(OldTopic)
except UnmappedClassError:
mapper(OldTopic, source_topics_table)
source_songs = self.source_session.query(OldSong).all()
if self.import_wizard:
self.import_wizard.progress_bar.setMaximum(len(source_songs))
for song in source_songs:
new_song = Song()
new_song.title = song.title
if has_media_files and hasattr(song, 'alternate_title'):
new_song.alternate_title = song.alternate_title
else:
old_titles = song.search_title.split('@')
if len(old_titles) > 1:
new_song.alternate_title = old_titles[1]
# Values will be set when cleaning the song.
if hasattr(song, 'song_key'):
new_song.song_key = song.song_key
if hasattr(song, 'transpose_by'):
new_song.transpose_by = song.transpose_by
new_song.search_title = ''
new_song.search_lyrics = ''
new_song.song_number = song.song_number
new_song.lyrics = song.lyrics
new_song.verse_order = song.verse_order
new_song.copyright = song.copyright
new_song.comments = song.comments
new_song.theme_name = song.theme_name
new_song.ccli_number = song.ccli_number
for author in song.authors:
existing_author = self.manager.get_object_filtered(Author, Author.display_name == author.display_name)
if existing_author is None:
existing_author = Author.populate(
first_name=author.first_name,
last_name=author.last_name,
display_name=author.display_name)
new_song.add_author(existing_author)
if song.book:
existing_song_book = self.manager.get_object_filtered(Book, Book.name == song.book.name)
if existing_song_book is None:
existing_song_book = Book.populate(name=song.book.name, publisher=song.book.publisher)
new_song.book = existing_song_book
if song.topics:
for topic in song.topics:
existing_topic = self.manager.get_object_filtered(Topic, Topic.name == topic.name)
if existing_topic is None:
existing_topic = Topic.populate(name=topic.name)
new_song.topics.append(existing_topic)
if has_media_files:
if song.media_files:
for media_file in song.media_files:
existing_media_file = self.manager.get_object_filtered(
MediaFile, MediaFile.file_name == media_file.file_name)
if existing_media_file:
new_song.media_files.append(existing_media_file)
else:
new_song.media_files.append(MediaFile.populate(file_name=media_file.file_name))
clean_song(self.manager, new_song)
self.manager.save_object(new_song)
if progress_dialog:
progress_dialog.setValue(progress_dialog.value() + 1)
progress_dialog.setLabelText(WizardStrings.ImportingType % new_song.title)
else:
self.import_wizard.increment_progress_bar(WizardStrings.ImportingType % new_song.title)
if self.stop_import_flag:
break
self.source_session.close()
engine.dispose()
def _process_authors(self, foilpresenterfolie, song):
"""
Adds the authors specified in the XML to the song.
:param foilpresenterfolie: The property object (lxml.objectify.ObjectifiedElement).
:param song: The song object.
"""
authors = []
try:
copyright = self._child(foilpresenterfolie.copyright.text_)
except AttributeError:
copyright = None
if copyright:
strings = []
if copyright.find('Copyright') != -1:
temp = copyright.partition('Copyright')
copyright = temp[0]
elif copyright.find('copyright') != -1:
temp = copyright.partition('copyright')
copyright = temp[0]
elif copyright.find('©') != -1:
temp = copyright.partition('©')
copyright = temp[0]
elif copyright.find('(c)') != -1:
temp = copyright.partition('(c)')
copyright = temp[0]
elif copyright.find('(C)') != -1:
temp = copyright.partition('(C)')
copyright = temp[0]
elif copyright.find('c)') != -1:
temp = copyright.partition('c)')
copyright = temp[0]
elif copyright.find('C)') != -1:
temp = copyright.partition('C)')
copyright = temp[0]
elif copyright.find('C:') != -1:
temp = copyright.partition('C:')
copyright = temp[0]
elif copyright.find('C,)') != -1:
temp = copyright.partition('C,)')
copyright = temp[0]
copyright = re.compile('\\n').sub(' ', copyright)
copyright = re.compile('\(.*\)').sub('', copyright)
if copyright.find('Rechte') != -1:
temp = copyright.partition('Rechte')
copyright = temp[0]
markers = ['Text +u\.?n?d? +Melodie[\w\,\. ]*:',
'Text +u\.?n?d? +Musik', 'T & M', 'Melodie und Satz',
'Text[\w\,\. ]*:', 'Melodie', 'Musik', 'Satz',
'Weise', '[dD]eutsch', '[dD]t[\.\:]', 'Englisch',
'[oO]riginal', 'Bearbeitung', '[R|r]efrain']
for marker in markers:
copyright = re.compile(marker).sub('<marker>', copyright, re.U)
copyright = re.compile('(?<=<marker>) *:').sub('', copyright)
x = 0
while True:
if copyright.find('<marker>') != -1:
temp = copyright.partition('<marker>')
if temp[0].strip() and x > 0:
strings.append(temp[0])
copyright = temp[2]
x += 1
elif x > 0:
strings.append(copyright)
break
else:
break
author_temp = []
for author in strings:
temp = re.split(',(?=\D{2})|(?<=\D),|\/(?=\D{3,})|(?<=\D);', author)
for tempx in temp:
author_temp.append(tempx)
for author in author_temp:
regex = '^[\/,;\-\s\.]+|[\/,;\-\s\.]+$|\s*[0-9]{4}\s*[\-\/]?\s*([0-9]{4})?[\/,;\-\s\.]*$'
author = re.compile(regex).sub('', author)
author = re.compile('[0-9]{1,2}\.\s?J(ahr)?h\.|um\s*$|vor\s*$').sub('', author)
author = re.compile('[N|n]ach.*$').sub('', author)
author = author.strip()
if re.search('\w+\.?\s+\w{3,}\s+[a|u]nd\s|\w+\.?\s+\w{3,}\s+&\s', author, re.U):
temp = re.split('\s[a|u]nd\s|\s&\s', author)
for tempx in temp:
tempx = tempx.strip()
authors.append(tempx)
elif len(author) > 2:
authors.append(author)
for display_name in authors:
author = self.manager.get_object_filtered(Author, Author.display_name == display_name)
if author is None:
# We need to create a new author, as the author does not exist.
author = Author.populate(display_name=display_name, last_name=display_name.split(' ')[-1],
first_name=' '.join(display_name.split(' ')[:-1]))
self.manager.save_object(author)
song.add_author(author)
def finish(self):
"""
All fields have been set to this song. Write the song to disk.
"""
if not self.checkComplete():
self.setDefaults()
return False
log.info('committing song %s to database', self.title)
song = Song()
song.title = self.title
if self.import_wizard is not None:
self.import_wizard.increment_progress_bar(WizardStrings.ImportingType % song.title)
song.alternate_title = self.alternate_title
# Values will be set when cleaning the song.
song.search_title = ''
song.search_lyrics = ''
song.verse_order = ''
song.song_number = self.songNumber
verses_changed_to_other = {}
sxml = SongXML()
other_count = 1
for (verse_def, verse_text, lang) in self.verses:
if verse_def[0].lower() in VerseType.tags:
verse_tag = verse_def[0].lower()
else:
new_verse_def = '%s%d' % (VerseType.tags[VerseType.Other], other_count)
verses_changed_to_other[verse_def] = new_verse_def
other_count += 1
verse_tag = VerseType.tags[VerseType.Other]
log.info('Versetype %s changing to %s', verse_def, new_verse_def)
verse_def = new_verse_def
sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], verse_text, lang)
song.lyrics = str(sxml.extract_xml(), 'utf-8')
if not self.verseOrderList and self.verseOrderListGeneratedUseful:
self.verseOrderList = self.verseOrderListGenerated
self.verseOrderList = [verses_changed_to_other.get(v, v) for v in self.verseOrderList]
song.verse_order = ' '.join(self.verseOrderList)
song.copyright = self.copyright
song.comments = self.comments
song.theme_name = self.themeName
song.ccli_number = self.ccliNumber
for authortext in self.authors:
author = self.manager.get_object_filtered(Author, Author.display_name == authortext)
if not author:
author = Author.populate(display_name=authortext,
last_name=authortext.split(' ')[-1],
first_name=' '.join(authortext.split(' ')[:-1]))
song.authors.append(author)
if self.songBookName:
song_book = self.manager.get_object_filtered(Book, Book.name == self.songBookName)
if song_book is None:
song_book = Book.populate(name=self.songBookName, publisher=self.songBookPub)
song.book = song_book
for topictext in self.topics:
if not topictext:
continue
topic = self.manager.get_object_filtered(Topic, Topic.name == topictext)
if topic is None:
topic = Topic.populate(name=topictext)
song.topics.append(topic)
# We need to save the song now, before adding the media files, so that
# we know where to save the media files to.
clean_song(self.manager, song)
self.manager.save_object(song)
# Now loop through the media files, copy them to the correct location,
# and save the song again.
for filename, weight in self.mediaFiles:
media_file = self.manager.get_object_filtered(MediaFile, MediaFile.file_name == filename)
if not media_file:
if os.path.dirname(filename):
filename = self.copyMediaFile(song.id, filename)
song.media_files.append(MediaFile.populate(file_name=filename, weight=weight))
self.manager.save_object(song)
self.setDefaults()
return True
def _process_authors(self, foilpresenterfolie, song):
"""
Adds the authors specified in the XML to the song.
:param foilpresenterfolie: The property object (lxml.objectify.ObjectifiedElement).
:param song: The song object.
"""
authors = []
try:
copyright = to_str(foilpresenterfolie.copyright.text_)
except AttributeError:
copyright = None
if copyright:
strings = []
if copyright.find('Copyright') != -1:
temp = copyright.partition('Copyright')
copyright = temp[0]
elif copyright.find('copyright') != -1:
temp = copyright.partition('copyright')
copyright = temp[0]
elif copyright.find('©') != -1:
temp = copyright.partition('©')
copyright = temp[0]
elif copyright.find('(c)') != -1:
temp = copyright.partition('(c)')
copyright = temp[0]
elif copyright.find('(C)') != -1:
temp = copyright.partition('(C)')
copyright = temp[0]
elif copyright.find('c)') != -1:
temp = copyright.partition('c)')
copyright = temp[0]
elif copyright.find('C)') != -1:
temp = copyright.partition('C)')
copyright = temp[0]
elif copyright.find('C:') != -1:
temp = copyright.partition('C:')
copyright = temp[0]
elif copyright.find('C,)') != -1:
temp = copyright.partition('C,)')
copyright = temp[0]
copyright = re.compile(r'\\n').sub(' ', copyright)
copyright = re.compile(r'\(.*\)').sub('', copyright)
if copyright.find('Rechte') != -1:
temp = copyright.partition('Rechte')
copyright = temp[0]
markers = [
r'Text +u\.?n?d? +Melodie[\w\,\. ]*:',
r'Text +u\.?n?d? +Musik', 'T & M', 'Melodie und Satz',
r'Text[\w\,\. ]*:', 'Melodie', 'Musik', 'Satz', 'Weise',
'[dD]eutsch', r'[dD]t[\.\:]', 'Englisch', '[oO]riginal',
'Bearbeitung', '[R|r]efrain'
]
for marker in markers:
copyright = re.compile(marker).sub('<marker>', copyright, re.U)
copyright = re.compile('(?<=<marker>) *:').sub('', copyright)
x = 0
while True:
if copyright.find('<marker>') != -1:
temp = copyright.partition('<marker>')
if temp[0].strip() and x > 0:
strings.append(temp[0])
copyright = temp[2]
x += 1
elif x > 0:
strings.append(copyright)
break
else:
break
author_temp = []
for author in strings:
temp = re.split(r',(?=\D{2})|(?<=\D),|\/(?=\D{3,})|(?<=\D);',
author)
for tempx in temp:
author_temp.append(tempx)
for author in author_temp:
regex = r'^[\/,;\-\s\.]+|[\/,;\-\s\.]+$|\s*[0-9]{4}\s*[\-\/]?\s*([0-9]{4})?[\/,;\-\s\.]*$'
author = re.compile(regex).sub('', author)
author = re.compile(
r'[0-9]{1,2}\.\s?J(ahr)?h\.|um\s*$|vor\s*$').sub(
'', author)
author = re.compile(r'[N|n]ach.*$').sub('', author)
author = author.strip()
if re.search(
r'\w+\.?\s+\w{3,}\s+[a|u]nd\s|\w+\.?\s+\w{3,}\s+&\s',
author, re.U):
temp = re.split(r'\s[a|u]nd\s|\s&\s', author)
for tempx in temp:
tempx = tempx.strip()
authors.append(tempx)
elif len(author) > 2:
authors.append(author)
for display_name in authors:
author = self.manager.get_object_filtered(
Author, Author.display_name == display_name)
if author is None:
# We need to create a new author, as the author does not exist.
author = Author.populate(display_name=display_name,
last_name=display_name.split(' ')[-1],
first_name=' '.join(
display_name.split(' ')[:-1]))
self.manager.save_object(author)
song.add_author(author)
def finish(self):
"""
All fields have been set to this song. Write the song to disk.
"""
if not self.check_complete():
self.set_defaults()
return False
log.info(
'committing song {title} to database'.format(title=self.title))
song = Song()
song.title = self.title
if self.import_wizard is not None:
# TODO: Verify format() with template variables
self.import_wizard.increment_progress_bar(
WizardStrings.ImportingType % song.title)
song.alternate_title = self.alternate_title
# Values will be set when cleaning the song.
song.search_title = ''
song.search_lyrics = ''
song.verse_order = ''
song.song_number = self.song_number
verses_changed_to_other = {}
sxml = SongXML()
other_count = 1
for (verse_def, verse_text, lang) in self.verses:
if verse_def[0].lower() in VerseType.tags:
verse_tag = verse_def[0].lower()
else:
new_verse_def = '{tag}{count:d}'.format(
tag=VerseType.tags[VerseType.Other], count=other_count)
verses_changed_to_other[verse_def] = new_verse_def
other_count += 1
verse_tag = VerseType.tags[VerseType.Other]
log.info('Versetype {old} changing to {new}'.format(
old=verse_def, new=new_verse_def))
verse_def = new_verse_def
sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], verse_text,
lang)
song.lyrics = str(sxml.extract_xml(), 'utf-8')
if not self.verse_order_list and self.verse_order_list_generated_useful:
self.verse_order_list = self.verse_order_list_generated
self.verse_order_list = [
verses_changed_to_other.get(v, v) for v in self.verse_order_list
]
song.verse_order = ' '.join(self.verse_order_list)
song.copyright = self.copyright
song.comments = self.comments
song.theme_name = self.theme_name
song.ccli_number = self.ccli_number
for author_text, author_type in self.authors:
author = self.manager.get_object_filtered(
Author, Author.display_name == author_text)
if not author:
author = Author.populate(display_name=author_text,
last_name=author_text.split(' ')[-1],
first_name=' '.join(
author_text.split(' ')[:-1]))
song.add_author(author, author_type)
if self.song_book_name:
song_book = self.manager.get_object_filtered(
Book, Book.name == self.song_book_name)
if song_book is None:
song_book = Book.populate(name=self.song_book_name,
publisher=self.song_book_pub)
song.add_songbook_entry(song_book, song.song_number)
for topic_text in self.topics:
if not topic_text:
continue
topic = self.manager.get_object_filtered(Topic,
Topic.name == topic_text)
if topic is None:
topic = Topic.populate(name=topic_text)
song.topics.append(topic)
# We need to save the song now, before adding the media files, so that
# we know where to save the media files to.
clean_song(self.manager, song)
self.manager.save_object(song)
# Now loop through the media files, copy them to the correct location,
# and save the song again.
for filename, weight in self.media_files:
media_file = self.manager.get_object_filtered(
MediaFile, MediaFile.file_name == filename)
if not media_file:
if os.path.dirname(filename):
filename = self.copy_media_file(song.id, filename)
song.media_files.append(
MediaFile.populate(file_name=filename, weight=weight))
self.manager.save_object(song)
self.set_defaults()
return True
def do_import(self, progress_dialog=None):
"""
Run the import for an OpenLP version 2 song database.
:param progress_dialog: The QProgressDialog used when importing songs from the FRW.
"""
class OldAuthor(BaseModel):
"""
Maps to the authors table
"""
pass
class OldBook(BaseModel):
"""
Maps to the songbooks table
"""
pass
class OldMediaFile(BaseModel):
"""
Maps to the media_files table
"""
pass
class OldSong(BaseModel):
"""
Maps to the songs table
"""
pass
class OldTopic(BaseModel):
"""
Maps to the topics table
"""
pass
class OldSongBookEntry(BaseModel):
"""
Maps to the songs_songbooks table
"""
pass
# Check the file type
if not isinstance(self.import_source, str) or not self.import_source.endswith('.sqlite'):
self.log_error(self.import_source, translate('SongsPlugin.OpenLPSongImport',
'Not a valid OpenLP 2 song database.'))
return
self.import_source = 'sqlite:///{url}'.format(url=self.import_source)
# Load the db file and reflect it
engine = create_engine(self.import_source)
source_meta = MetaData()
source_meta.reflect(engine)
self.source_session = scoped_session(sessionmaker(bind=engine))
# Run some checks to see which version of the database we have
if 'media_files' in list(source_meta.tables.keys()):
has_media_files = True
else:
has_media_files = False
if 'songs_songbooks' in list(source_meta.tables.keys()):
has_songs_books = True
else:
has_songs_books = False
# Load up the tabls and map them out
source_authors_table = source_meta.tables['authors']
source_song_books_table = source_meta.tables['song_books']
source_songs_table = source_meta.tables['songs']
source_topics_table = source_meta.tables['topics']
source_authors_songs_table = source_meta.tables['authors_songs']
source_songs_topics_table = source_meta.tables['songs_topics']
source_media_files_songs_table = None
# Set up media_files relations
if has_media_files:
source_media_files_table = source_meta.tables['media_files']
source_media_files_songs_table = source_meta.tables.get('media_files_songs')
try:
class_mapper(OldMediaFile)
except UnmappedClassError:
mapper(OldMediaFile, source_media_files_table)
if has_songs_books:
source_songs_songbooks_table = source_meta.tables['songs_songbooks']
try:
class_mapper(OldSongBookEntry)
except UnmappedClassError:
mapper(OldSongBookEntry, source_songs_songbooks_table, properties={'songbook': relation(OldBook)})
# Set up the songs relationships
song_props = {
'authors': relation(OldAuthor, backref='songs', secondary=source_authors_songs_table),
'topics': relation(OldTopic, backref='songs', secondary=source_songs_topics_table)
}
if has_media_files:
if isinstance(source_media_files_songs_table, Table):
song_props['media_files'] = relation(OldMediaFile, backref='songs',
secondary=source_media_files_songs_table)
else:
song_props['media_files'] = \
relation(OldMediaFile, backref='songs',
foreign_keys=[source_media_files_table.c.song_id],
primaryjoin=source_songs_table.c.id == source_media_files_table.c.song_id)
if has_songs_books:
song_props['songbook_entries'] = relation(OldSongBookEntry, backref='song', cascade='all, delete-orphan')
else:
song_props['book'] = relation(OldBook, backref='songs')
# Map the rest of the tables
try:
class_mapper(OldAuthor)
except UnmappedClassError:
mapper(OldAuthor, source_authors_table)
try:
class_mapper(OldBook)
except UnmappedClassError:
mapper(OldBook, source_song_books_table)
try:
class_mapper(OldSong)
except UnmappedClassError:
mapper(OldSong, source_songs_table, properties=song_props)
try:
class_mapper(OldTopic)
except UnmappedClassError:
mapper(OldTopic, source_topics_table)
source_songs = self.source_session.query(OldSong).all()
if self.import_wizard:
self.import_wizard.progress_bar.setMaximum(len(source_songs))
for song in source_songs:
new_song = Song()
new_song.title = song.title
if has_media_files and hasattr(song, 'alternate_title'):
new_song.alternate_title = song.alternate_title
else:
old_titles = song.search_title.split('@')
if len(old_titles) > 1:
new_song.alternate_title = old_titles[1]
# Transfer the values to the new song object
new_song.search_title = ''
new_song.search_lyrics = ''
new_song.lyrics = song.lyrics
new_song.verse_order = song.verse_order
new_song.copyright = song.copyright
new_song.comments = song.comments
new_song.theme_name = song.theme_name
new_song.ccli_number = song.ccli_number
if hasattr(song, 'song_number') and song.song_number:
new_song.song_number = song.song_number
# Find or create all the authors and add them to the new song object
for author in song.authors:
existing_author = self.manager.get_object_filtered(Author, Author.display_name == author.display_name)
if not existing_author:
existing_author = Author.populate(
first_name=author.first_name,
last_name=author.last_name,
display_name=author.display_name)
new_song.add_author(existing_author)
# Find or create all the topics and add them to the new song object
if song.topics:
for topic in song.topics:
existing_topic = self.manager.get_object_filtered(Topic, Topic.name == topic.name)
if not existing_topic:
existing_topic = Topic.populate(name=topic.name)
new_song.topics.append(existing_topic)
# Find or create all the songbooks and add them to the new song object
if has_songs_books and song.songbook_entries:
for entry in song.songbook_entries:
existing_book = self.manager.get_object_filtered(Book, Book.name == entry.songbook.name)
if not existing_book:
existing_book = Book.populate(name=entry.songbook.name, publisher=entry.songbook.publisher)
new_song.add_songbook_entry(existing_book, entry.entry)
elif song.book:
existing_book = self.manager.get_object_filtered(Book, Book.name == song.book.name)
if not existing_book:
existing_book = Book.populate(name=song.book.name, publisher=song.book.publisher)
new_song.add_songbook_entry(existing_book, '')
# Find or create all the media files and add them to the new song object
if has_media_files and song.media_files:
for media_file in song.media_files:
existing_media_file = self.manager.get_object_filtered(
MediaFile, MediaFile.file_name == media_file.file_name)
if existing_media_file:
new_song.media_files.append(existing_media_file)
else:
new_song.media_files.append(MediaFile.populate(file_name=media_file.file_name))
clean_song(self.manager, new_song)
self.manager.save_object(new_song)
if progress_dialog:
progress_dialog.setValue(progress_dialog.value() + 1)
# TODO: Verify format() with template strings
progress_dialog.setLabelText(WizardStrings.ImportingType % new_song.title)
else:
# TODO: Verify format() with template strings
self.import_wizard.increment_progress_bar(WizardStrings.ImportingType % new_song.title)
if self.stop_import_flag:
break
self.source_session.close()
engine.dispose()