def save_song(self, song):
"""
Save a song to the database, using the db_manager
:param song:
:return:
"""
db_song = Song.populate(title=song['title'], copyright=song['copyright'], ccli_number=song['ccli_number'])
song_xml = SongXML()
verse_order = []
for verse in song['verses']:
verse_type, verse_number = verse['label'].split(' ')[:2]
verse_type = VerseType.from_loose_input(verse_type)
verse_number = int(verse_number)
song_xml.add_verse_to_lyrics(VerseType.tags[verse_type], verse_number, verse['lyrics'])
verse_order.append('%s%s' % (VerseType.tags[verse_type], verse_number))
db_song.verse_order = ' '.join(verse_order)
db_song.lyrics = song_xml.extract_xml()
clean_song(self.db_manager, db_song)
self.db_manager.save_object(db_song)
db_song.authors_songs = []
for author_name in song['authors']:
author = self.db_manager.get_object_filtered(Author, Author.display_name == author_name)
if not author:
author = Author.populate(first_name=author_name.rsplit(' ', 1)[0],
last_name=author_name.rsplit(' ', 1)[1],
display_name=author_name)
db_song.add_author(author)
self.db_manager.save_object(db_song)
return db_song
def on_verse_edit_all_button_clicked(self):
verse_list = ''
if self.verse_list_widget.rowCount() > 0:
for row in range(self.verse_list_widget.rowCount()):
item = self.verse_list_widget.item(row, 0)
field = item.data(QtCore.Qt.UserRole)
verse_tag = VerseType.translated_name(field[0])
verse_num = field[1:]
verse_list += '---[%s:%s]---\n' % (verse_tag, verse_num)
verse_list += item.text()
verse_list += '\n'
self.verse_form.set_verse(verse_list)
else:
self.verse_form.set_verse('')
if not self.verse_form.exec_():
return
verse_list = self.verse_form.get_all_verses()
verse_list = str(verse_list.replace('\r\n', '\n'))
self.verse_list_widget.clear()
self.verse_list_widget.setRowCount(0)
for row in self.find_verse_split.split(verse_list):
for match in row.split('---['):
for count, parts in enumerate(match.split(']---\n')):
if count == 0:
if len(parts) == 0:
continue
# handling carefully user inputted versetags
separator = parts.find(':')
if separator >= 0:
verse_name = parts[0:separator].strip()
verse_num = parts[separator+1:].strip()
else:
verse_name = parts
verse_num = '1'
verse_index = VerseType.from_loose_input(verse_name)
verse_tag = VerseType.tags[verse_index]
# Later we need to handle v1a as well.
#regex = re.compile(r'(\d+\w.)')
regex = re.compile(r'\D*(\d+)\D*')
match = regex.match(verse_num)
if match:
verse_num = match.group(1)
else:
verse_num = '1'
verse_def = '%s%s' % (verse_tag, verse_num)
else:
if parts.endswith('\n'):
parts = parts.rstrip('\n')
item = QtGui.QTableWidgetItem(parts)
item.setData(QtCore.Qt.UserRole, verse_def)
self.verse_list_widget.setRowCount(self.verse_list_widget.rowCount() + 1)
self.verse_list_widget.setItem(self.verse_list_widget.rowCount() - 1, 0, item)
self.tag_rows()
self.verse_edit_button.setEnabled(False)
self.verse_delete_button.setEnabled(False)
# Check if all verse tags are used.
self.on_verse_order_text_changed(self.verse_order_edit.text())
def test_from_loose_input_with_valid_input(self, mocked_translated_tags):
"""
Test that the from_loose_input() method returns valid output on valid input.
"""
# GIVEN: A mocked VerseType.translated_tags
# WHEN: We run the from_loose_input() method with a valid verse type, we get the expected VerseType back
result = VerseType.from_loose_input('v')
# THEN: The result should be a Verse
self.assertEqual(result, VerseType.Verse, 'The result should be a verse, but was "%s"' % result)
def test_from_loose_input_with_invalid_input(self, mocked_translated_tags):
"""
Test that the from_loose_input() method returns a sane default when passed an invalid tag and None as default.
"""
# GIVEN: A mocked VerseType.translated_tags
# WHEN: We run the from_loose_input() method with an invalid verse type, we get the specified default back
result = VerseType.from_loose_input('m', None)
# THEN: The result should be None
self.assertIsNone(result, 'The result should be None, but was "%s"' % result)
def test_from_loose_input_with_valid_input(self, mocked_translated_tags):
"""
Test that the from_loose_input() method returns valid output on valid input.
"""
# GIVEN: A mocked VerseType.translated_tags
# WHEN: We run the from_loose_input() method with a valid verse type, we get the expected VerseType back
result = VerseType.from_loose_input('v')
# THEN: The result should be a Verse
assert result == VerseType.Verse, 'The result should be a verse, but was "%s"' % result
def test_from_loose_input_with_invalid_input(self, mocked_translated_tags):
"""
Test that the from_loose_input() method returns a sane default when passed an invalid tag and None as default.
"""
# GIVEN: A mocked VerseType.translated_tags
# WHEN: We run the from_loose_input() method with an invalid verse type, we get the specified default back
result = VerseType.from_loose_input('m', None)
# THEN: The result should be None
assert result is None, 'The result should be None, but was "%s"' % result
def save_song(self, song):
"""
Save a song to the database, using the db_manager
:param song:
:return:
"""
db_song = Song.populate(title=song['title'],
copyright=song['copyright'],
ccli_number=song['ccli_number'])
song_xml = SongXML()
verse_order = []
for verse in song['verses']:
if ' ' in verse['label']:
verse_type, verse_number = verse['label'].split(' ', 1)
else:
verse_type = verse['label']
verse_number = 1
verse_type = VerseType.from_loose_input(verse_type)
verse_number = int(verse_number)
song_xml.add_verse_to_lyrics(VerseType.tags[verse_type],
verse_number, verse['lyrics'])
verse_order.append('%s%s' %
(VerseType.tags[verse_type], verse_number))
db_song.verse_order = ' '.join(verse_order)
db_song.lyrics = song_xml.extract_xml()
clean_song(self.db_manager, db_song)
self.db_manager.save_object(db_song)
db_song.authors_songs = []
for author_name in song['authors']:
author = self.db_manager.get_object_filtered(
Author, Author.display_name == author_name)
if not author:
name_parts = author_name.rsplit(' ', 1)
first_name = name_parts[0]
if len(name_parts) == 1:
last_name = ''
else:
last_name = name_parts[1]
author = Author.populate(first_name=first_name,
last_name=last_name,
display_name=author_name)
db_song.add_author(author)
db_song.topics = []
for topic_name in song.get('topics', []):
topic = self.db_manager.get_object_filtered(
Topic, Topic.name == topic_name)
if not topic:
topic = Topic.populate(name=topic_name)
db_song.topics.append(topic)
self.db_manager.save_object(db_song)
return db_song
def save_song(self, song):
"""
Save a song to the database, using the db_manager
:param song:
:return:
"""
db_song = Song.populate(title=song['title'], copyright=song['copyright'], ccli_number=song['ccli_number'])
song_xml = SongXML()
verse_order = []
for verse in song['verses']:
if ' ' in verse['label']:
verse_type, verse_number = verse['label'].split(' ', 1)
else:
verse_type = verse['label']
verse_number = 1
verse_type = VerseType.from_loose_input(verse_type)
verse_number = int(verse_number)
song_xml.add_verse_to_lyrics(VerseType.tags[verse_type], verse_number, verse['lyrics'])
verse_order.append('{tag}{number}'.format(tag=VerseType.tags[verse_type], number=verse_number))
db_song.verse_order = ' '.join(verse_order)
db_song.lyrics = song_xml.extract_xml()
clean_song(self.db_manager, db_song)
self.db_manager.save_object(db_song)
db_song.authors_songs = []
for author_name in song['authors']:
author = self.db_manager.get_object_filtered(Author, Author.display_name == author_name)
if not author:
name_parts = author_name.rsplit(' ', 1)
first_name = name_parts[0]
if len(name_parts) == 1:
last_name = ''
else:
last_name = name_parts[1]
author = Author.populate(first_name=first_name, last_name=last_name, display_name=author_name)
db_song.add_author(author)
for topic_name in song.get('topics', []):
topic = self.db_manager.get_object_filtered(Topic, Topic.name == topic_name)
if not topic:
topic = Topic.populate(name=topic_name)
db_song.topics.append(topic)
self.db_manager.save_object(db_song)
return db_song
def save_song(self, song):
"""
Save a song to the database, using the db_manager
:param song:
:return:
"""
db_song = Song.populate(title=song['title'],
copyright=song['copyright'],
ccli_number=song['ccli_number'])
song_xml = SongXML()
verse_order = []
for verse in song['verses']:
verse_type, verse_number = verse['label'].split(' ')[:2]
verse_type = VerseType.from_loose_input(verse_type)
verse_number = int(verse_number)
song_xml.add_verse_to_lyrics(VerseType.tags[verse_type],
verse_number, verse['lyrics'])
verse_order.append('%s%s' %
(VerseType.tags[verse_type], verse_number))
db_song.verse_order = ' '.join(verse_order)
db_song.lyrics = song_xml.extract_xml()
clean_song(self.db_manager, db_song)
self.db_manager.save_object(db_song)
db_song.authors_songs = []
for author_name in song['authors']:
author = self.db_manager.get_object_filtered(
Author, Author.display_name == author_name)
if not author:
author = Author.populate(first_name=author_name.rsplit(' ',
1)[0],
last_name=author_name.rsplit(' ',
1)[1],
display_name=author_name)
db_song.add_author(author)
self.db_manager.save_object(db_song)
return db_song
def do_import(self):
"""
Receive a CSV file to import.
"""
# Get encoding
encoding = get_file_encoding(self.import_source)['encoding']
with self.import_source.open('r', encoding=encoding) as songs_file:
songs_reader = csv.DictReader(songs_file, escapechar='\\')
try:
records = list(songs_reader)
except csv.Error as e:
self.log_error(
translate('SongsPlugin.WorshipAssistantImport',
'Error reading CSV file.'),
translate('SongsPlugin.WorshipAssistantImport',
'Line {number:d}: {error}').format(
number=songs_reader.line_num, error=e))
return
num_records = len(records)
log.info('{count} records found in CSV file'.format(count=num_records))
self.import_wizard.progress_bar.setMaximum(num_records)
# Create regex to strip html tags
re_html_strip = re.compile(r'<[^>]+>')
for index, record in enumerate(records, 1):
if self.stop_import_flag:
return
# Ensure that all keys are uppercase
record = dict(
(field.upper(), value) for field, value in record.items())
# The CSV file has a line in the middle of the file where the headers are repeated.
# We need to skip this line.
if record['TITLE'] == "TITLE" and record[
'AUTHOR'] == 'AUTHOR' and record['LYRICS2'] == 'LYRICS2':
continue
self.set_defaults()
verse_order_list = []
try:
self.title = record['TITLE']
if record['AUTHOR'] != EMPTY_STR:
self.parse_author(record['AUTHOR'])
if record['COPYRIGHT'] != EMPTY_STR:
self.add_copyright(record['COPYRIGHT'])
if record['CCLINR'] != EMPTY_STR:
self.ccli_number = record['CCLINR']
if record['ROADMAP'] != EMPTY_STR:
verse_order_list = [
x.strip() for x in record['ROADMAP'].split(',')
]
lyrics = record['LYRICS2']
except UnicodeDecodeError as e:
self.log_error(
translate('SongsPlugin.WorshipAssistantImport',
'Record {count:d}').format(count=index),
translate('SongsPlugin.WorshipAssistantImport',
'Decoding error: {error}').format(error=e))
continue
except TypeError as e:
self.log_error(
translate('SongsPlugin.WorshipAssistantImport',
'File not valid WorshipAssistant CSV format.'),
'TypeError: {error}'.format(error=e))
return
verse = ''
used_verses = []
verse_id = VerseType.tags[VerseType.Verse] + '1'
for line in lyrics.splitlines():
if line.startswith('['): # verse marker
# Add previous verse
if verse:
# remove trailing linebreak, part of the WA syntax
self.add_verse(verse[:-1], verse_id)
used_verses.append(verse_id)
verse = ''
# drop the square brackets
right_bracket = line.find(']')
content = line[1:right_bracket].lower()
match = re.match(r'(\D*)(\d+)', content)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
else:
# otherwise we assume number 1 and take the whole prefix as the verse tag
verse_tag = content
verse_num = '1'
verse_index = VerseType.from_loose_input(
verse_tag) if verse_tag else 0
verse_tag = VerseType.tags[verse_index]
# Update verse order when the verse name has changed
verse_id = verse_tag + verse_num
# Make sure we've not choosen an id already used
while verse_id in verse_order_list and content in verse_order_list:
verse_num = str(int(verse_num) + 1)
verse_id = verse_tag + verse_num
if content != verse_id:
for i in range(len(verse_order_list)):
if verse_order_list[i].lower() == content.lower():
verse_order_list[i] = verse_id
else:
# add line text to verse. Strip out html
verse += re_html_strip.sub('', line) + '\n'
if verse:
# remove trailing linebreak, part of the WA syntax
if verse.endswith('\n\n'):
verse = verse[:-1]
self.add_verse(verse, verse_id)
used_verses.append(verse_id)
if verse_order_list:
# Use the verse order in the import, but remove entries that doesn't have a text
cleaned_verse_order_list = []
for verse in verse_order_list:
if verse in used_verses:
cleaned_verse_order_list.append(verse)
self.verse_order_list = cleaned_verse_order_list
if not self.finish():
self.log_error(
translate('SongsPlugin.WorshipAssistantImport',
'Record {count:d}').format(count=index) +
(': "' + self.title + '"' if self.title else ''))
def do_import_file(self, file):
"""
Process the OpenSong file - pass in a file-like object, not a file path.
"""
self.set_defaults()
try:
tree = objectify.parse(file)
except (Error, LxmlError):
self.log_error(file.name, SongStrings.XMLSyntaxError)
log.exception('Error parsing XML')
return
root = tree.getroot()
if root.tag != 'song':
self.log_error(
file.name,
str(
translate(
'SongsPlugin.OpenSongImport',
'Invalid OpenSong song file. Missing song tag.')))
return
fields = dir(root)
decode = {
'copyright': self.add_copyright,
'ccli': 'ccli_number',
'author': self.parse_author,
'title': 'title',
'aka': 'alternate_title',
'hymn_number': self.parse_song_book_name_and_number,
'user1': self.add_comment,
'user2': self.add_comment,
'user3': self.add_comment
}
for attr, fn_or_string in list(decode.items()):
if attr in fields:
ustring = str(root.__getattr__(attr))
if isinstance(fn_or_string, str):
if attr in ['ccli']:
ustring = ''.join(re.findall('\d+', ustring))
if ustring:
setattr(self, fn_or_string, int(ustring))
else:
setattr(self, fn_or_string, None)
else:
setattr(self, fn_or_string, ustring)
else:
fn_or_string(ustring)
# Themes look like "God: Awe/Wonder", but we just want
# "Awe" and "Wonder". We use a set to ensure each topic
# is only added once, in case it is already there, which
# is actually quite likely if the alttheme is set
topics = set(self.topics)
if 'theme' in fields:
theme = str(root.theme)
subthemes = theme[theme.find(':') + 1:].split('/')
for topic in subthemes:
topics.add(topic.strip())
if 'alttheme' in fields:
theme = str(root.alttheme)
subthemes = theme[theme.find(':') + 1:].split('/')
for topic in subthemes:
topics.add(topic.strip())
self.topics = list(topics)
self.topics.sort()
# data storage while importing
verses = {}
# keep track of verses appearance order
our_verse_order = []
# default verse
verse_tag = VerseType.tags[VerseType.Verse]
verse_num = '1'
# for the case where song has several sections with same marker
inst = 1
if 'lyrics' in fields:
lyrics = str(root.lyrics)
else:
lyrics = ''
for this_line in lyrics.split('\n'):
if not this_line.strip():
continue
# skip this line if it is a comment
if this_line.startswith(';'):
continue
# skip guitar chords and page and column breaks
if this_line.startswith('.') or this_line.startswith(
'---') or this_line.startswith('-!!'):
continue
# verse/chorus/etc. marker
if this_line.startswith('['):
# drop the square brackets
right_bracket = this_line.find(']')
content = this_line[1:right_bracket].lower()
# have we got any digits? If so, verse number is everything from the digits to the end (openlp does not
# have concept of part verses, so just ignore any non integers on the end (including floats))
match = re.match('(\D*)(\d+)', content)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
else:
# otherwise we assume number 1 and take the whole prefix as the verse tag
verse_tag = content
verse_num = '1'
verse_index = VerseType.from_loose_input(
verse_tag) if verse_tag else 0
verse_tag = VerseType.tags[verse_index]
inst = 1
if [verse_tag, verse_num, inst
] in our_verse_order and verse_num in verses.get(
verse_tag, {}):
inst = len(verses[verse_tag][verse_num]) + 1
continue
# number at start of line.. it's verse number
if this_line[0].isdigit():
verse_num = this_line[0]
this_line = this_line[1:].strip()
verses.setdefault(verse_tag, {})
verses[verse_tag].setdefault(verse_num, {})
if inst not in verses[verse_tag][verse_num]:
verses[verse_tag][verse_num][inst] = []
our_verse_order.append([verse_tag, verse_num, inst])
# Tidy text and remove the ____s from extended words
this_line = self.tidy_text(this_line)
this_line = this_line.replace('_', '')
this_line = this_line.replace('||', '\n[---]\n')
this_line = this_line.strip()
# If the line consists solely of a '|', then just use the implicit newline
# Otherwise, add a newline for each '|'
if this_line == '|':
this_line = ''
else:
this_line = this_line.replace('|', '\n')
verses[verse_tag][verse_num][inst].append(this_line)
# done parsing
# add verses in original order
verse_joints = {}
for (verse_tag, verse_num, inst) in our_verse_order:
lines = '\n'.join(verses[verse_tag][verse_num][inst])
length = 0
while length < len(verse_num) and verse_num[length].isnumeric():
length += 1
verse_def = '{tag}{number}'.format(tag=verse_tag,
number=verse_num[:length])
verse_joints[verse_def] = '{verse}\n[---]\n{lines}'.format(verse=verse_joints[verse_def], lines=lines) \
if verse_def in verse_joints else lines
# Parsing the dictionary produces the elements in a non-intuitive order. While it "works", it's not a
# natural layout should the user come back to edit the song. Instead we sort by the verse type, so that we
# get all the verses in order (v1, v2, ...), then the chorus(es), bridge(s), pre-chorus(es) etc. We use a
# tuple for the key, since tuples naturally sort in this manner.
verse_defs = sorted(
verse_joints.keys(),
key=lambda verse_def:
(VerseType.from_tag(verse_def[0]), int(verse_def[1:])))
for verse_def in verse_defs:
lines = verse_joints[verse_def]
self.add_verse(lines, verse_def)
if not self.verses:
self.add_verse('')
# figure out the presentation order, if present
if 'presentation' in fields and root.presentation:
order = str(root.presentation)
# We make all the tags in the lyrics lower case, so match that here and then split into a list on the
# whitespace.
order = order.lower().split()
for verse_def in order:
match = re.match('(\D*)(\d+.*)', verse_def)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
if not verse_tag:
verse_tag = VerseType.tags[VerseType.Verse]
else:
# Assume it's no.1 if there are no digits
verse_tag = verse_def
verse_num = '1'
verse_index = VerseType.from_loose_input(verse_tag)
verse_tag = VerseType.tags[verse_index]
verse_def = '{tag}{number}'.format(tag=verse_tag,
number=verse_num)
if verse_num in verses.get(verse_tag, {}):
self.verse_order_list.append(verse_def)
else:
log.info(
'Got order {order} but not in verse tags, dropping this item from presentation '
'order'.format(order=verse_def))
if not self.finish():
self.log_error(file.name)
def do_import(self):
"""
Receive a CSV file to import.
"""
# Get encoding
detect_file = open(self.import_source, 'rb')
detect_content = detect_file.read()
details = chardet.detect(detect_content)
detect_file.close()
songs_file = open(self.import_source, 'r', encoding=details['encoding'])
songs_reader = csv.DictReader(songs_file, escapechar='\\')
try:
records = list(songs_reader)
except csv.Error as e:
self.log_error(translate('SongsPlugin.WorshipAssistantImport', 'Error reading CSV file.'),
translate('SongsPlugin.WorshipAssistantImport', 'Line %d: %s') %
(songs_reader.line_num, e))
return
num_records = len(records)
log.info('%s records found in CSV file' % num_records)
self.import_wizard.progress_bar.setMaximum(num_records)
# Create regex to strip html tags
re_html_strip = re.compile(r'<[^>]+>')
for index, record in enumerate(records, 1):
if self.stop_import_flag:
return
# Ensure that all keys are uppercase
record = dict((field.upper(), value) for field, value in record.items())
# The CSV file has a line in the middle of the file where the headers are repeated.
# We need to skip this line.
if record['TITLE'] == "TITLE" and record['AUTHOR'] == 'AUTHOR' and record['LYRICS2'] == 'LYRICS2':
continue
self.set_defaults()
verse_order_list = []
try:
self.title = record['TITLE']
if record['AUTHOR'] != EMPTY_STR:
self.parse_author(record['AUTHOR'])
if record['COPYRIGHT'] != EMPTY_STR:
self.add_copyright(record['COPYRIGHT'])
if record['CCLINR'] != EMPTY_STR:
self.ccli_number = record['CCLINR']
if record['ROADMAP'] != EMPTY_STR:
verse_order_list = [x.strip() for x in record['ROADMAP'].split(',')]
lyrics = record['LYRICS2']
except UnicodeDecodeError as e:
self.log_error(translate('SongsPlugin.WorshipAssistantImport', 'Record %d' % index),
translate('SongsPlugin.WorshipAssistantImport', 'Decoding error: %s') % e)
continue
except TypeError as e:
self.log_error(translate('SongsPlugin.WorshipAssistantImport',
'File not valid WorshipAssistant CSV format.'), 'TypeError: %s' % e)
return
verse = ''
used_verses = []
verse_id = VerseType.tags[VerseType.Verse] + '1'
for line in lyrics.splitlines():
if line.startswith('['): # verse marker
# Add previous verse
if verse:
# remove trailing linebreak, part of the WA syntax
self.add_verse(verse[:-1], verse_id)
used_verses.append(verse_id)
verse = ''
# drop the square brackets
right_bracket = line.find(']')
content = line[1:right_bracket].lower()
match = re.match('(\D*)(\d+)', content)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
else:
# otherwise we assume number 1 and take the whole prefix as the verse tag
verse_tag = content
verse_num = '1'
verse_index = VerseType.from_loose_input(verse_tag) if verse_tag else 0
verse_tag = VerseType.tags[verse_index]
# Update verse order when the verse name has changed
verse_id = verse_tag + verse_num
# Make sure we've not choosen an id already used
while verse_id in verse_order_list and content in verse_order_list:
verse_num = str(int(verse_num) + 1)
verse_id = verse_tag + verse_num
if content != verse_id:
for i in range(len(verse_order_list)):
if verse_order_list[i].lower() == content.lower():
verse_order_list[i] = verse_id
else:
# add line text to verse. Strip out html
verse += re_html_strip.sub('', line) + '\n'
if verse:
# remove trailing linebreak, part of the WA syntax
if verse.endswith('\n\n'):
verse = verse[:-1]
self.add_verse(verse, verse_id)
used_verses.append(verse_id)
if verse_order_list:
# Use the verse order in the import, but remove entries that doesn't have a text
cleaned_verse_order_list = []
for verse in verse_order_list:
if verse in used_verses:
cleaned_verse_order_list.append(verse)
self.verse_order_list = cleaned_verse_order_list
if not self.finish():
self.log_error(translate('SongsPlugin.WorshipAssistantImport', 'Record %d') % index +
(': "' + self.title + '"' if self.title else ''))
songs_file.close()
def parse(self, data, cell=False):
"""
Process the records
:param data: The data to be processed
:param cell: ?
:return:
"""
if not cell and (len(data) == 0 or data[0:1] != b'['
or data.strip()[-1:] != b']'):
self.log_error('File is malformed')
return False
i = 1
verse_type = VerseType.tags[VerseType.Verse]
while i < len(data):
# Data is held as #name: value pairs inside groups marked as [].
# Now we are looking for the name.
if data[i:i + 1] == b'#':
name_end = data.find(b':', i + 1)
name = data[i + 1:name_end].decode(self.encoding).upper()
i = name_end + 1
while data[i:i + 1] == b' ':
i += 1
if data[i:i + 1] == b'"':
end = data.find(b'"', i + 1)
value = data[i + 1:end]
elif data[i:i + 1] == b'[':
j = i
inside_quotes = False
while j < len(data):
char = data[j:j + 1]
if char == b'"':
inside_quotes = not inside_quotes
elif not inside_quotes and char == b']':
end = j + 1
break
j += 1
value = data[i:end]
else:
end = data.find(b',', i + 1)
if data.find(b'(', i, end) != -1:
end = data.find(b')', i) + 1
value = data[i:end]
# If we are in the main group.
if not cell:
if name == 'TITLE':
self.title = self.decode(self.unescape(value))
elif name == 'AUTHOR':
author = self.decode(self.unescape(value))
if len(author):
self.add_author(author)
elif name == 'COPYRIGHT':
self.add_copyright(self.decode(self.unescape(value)))
elif name[0:4] == 'CELL':
self.parse(value, cell=name[4:])
# We are in a verse group.
else:
if name == 'MARKER_NAME':
value = self.decode(value).strip()
if len(value):
verse_type = VerseType.tags[
VerseType.from_loose_input(value[0])]
if len(value) >= 2 and value[-1] in [
'0', '1', '2', '3', '4', '5', '6', '7',
'8', '9'
]:
verse_type = "{verse}{value}".format(
verse=verse_type, value=value[-1])
elif name == 'HOTKEY':
value = self.decode(value).strip()
# HOTKEY always appears after MARKER_NAME, so it
# effectively overrides MARKER_NAME, if present.
if len(value) and value in list(
HOTKEY_TO_VERSE_TYPE.keys()):
verse_type = HOTKEY_TO_VERSE_TYPE[value]
if name == 'RTF':
value = self.unescape(value)
value = self.decode(value)
result = strip_rtf(value, self.encoding)
if result is None:
return False
verse, self.encoding = result
lines = verse.strip().split('\n')
# If any line inside any verse contains CCLI or
# only Public Domain, we treat this as special data:
# we remove that line and add data to specific field.
processed_lines = []
for i in range(len(lines)):
line = lines[i].strip()
if line[:3].lower() == 'ccl':
m = re.search(r'[0-9]+', line)
if m:
self.ccli_number = int(m.group(0))
continue
elif line.lower() == 'public domain':
self.add_copyright('Public Domain')
continue
processed_lines.append(line)
self.add_verse('\n'.join(processed_lines).strip(),
verse_type)
if end == -1:
break
i = end + 1
i += 1
return True
def do_import_file(self, file):
"""
Process the OpenSong file - pass in a file-like object, not a file path.
"""
self.set_defaults()
try:
tree = objectify.parse(file)
except (Error, LxmlError):
self.log_error(file.name, SongStrings.XMLSyntaxError)
log.exception('Error parsing XML')
return
root = tree.getroot()
if root.tag != 'song':
self.log_error(file.name, str(
translate('SongsPlugin.OpenSongImport', 'Invalid OpenSong song file. Missing song tag.')))
return
fields = dir(root)
decode = {
'copyright': self.add_copyright,
'ccli': 'ccli_number',
'author': self.parse_author,
'title': 'title',
'aka': 'alternate_title',
'hymn_number': self.parse_song_book_name_and_number,
'user1': self.add_comment,
'user2': self.add_comment,
'user3': self.add_comment
}
for attr, fn_or_string in list(decode.items()):
if attr in fields:
ustring = str(root.__getattr__(attr))
if isinstance(fn_or_string, str):
if attr in ['ccli']:
if ustring:
setattr(self, fn_or_string, int(ustring))
else:
setattr(self, fn_or_string, None)
else:
setattr(self, fn_or_string, ustring)
else:
fn_or_string(ustring)
# Themes look like "God: Awe/Wonder", but we just want
# "Awe" and "Wonder". We use a set to ensure each topic
# is only added once, in case it is already there, which
# is actually quite likely if the alttheme is set
topics = set(self.topics)
if 'theme' in fields:
theme = str(root.theme)
subthemes = theme[theme.find(':')+1:].split('/')
for topic in subthemes:
topics.add(topic.strip())
if 'alttheme' in fields:
theme = str(root.alttheme)
subthemes = theme[theme.find(':')+1:].split('/')
for topic in subthemes:
topics.add(topic.strip())
self.topics = list(topics)
self.topics.sort()
# data storage while importing
verses = {}
# keep track of verses appearance order
our_verse_order = []
# default verse
verse_tag = VerseType.tags[VerseType.Verse]
verse_num = '1'
# for the case where song has several sections with same marker
inst = 1
if 'lyrics' in fields:
lyrics = str(root.lyrics)
else:
lyrics = ''
for this_line in lyrics.split('\n'):
if not this_line.strip():
continue
# skip this line if it is a comment
if this_line.startswith(';'):
continue
# skip guitar chords and page and column breaks
if this_line.startswith('.') or this_line.startswith('---') or this_line.startswith('-!!'):
continue
# verse/chorus/etc. marker
if this_line.startswith('['):
# drop the square brackets
right_bracket = this_line.find(']')
content = this_line[1:right_bracket].lower()
# have we got any digits? If so, verse number is everything from the digits to the end (openlp does not
# have concept of part verses, so just ignore any non integers on the end (including floats))
match = re.match('(\D*)(\d+)', content)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
else:
# otherwise we assume number 1 and take the whole prefix as the verse tag
verse_tag = content
verse_num = '1'
verse_index = VerseType.from_loose_input(verse_tag) if verse_tag else 0
verse_tag = VerseType.tags[verse_index]
inst = 1
if [verse_tag, verse_num, inst] in our_verse_order and verse_num in verses.get(verse_tag, {}):
inst = len(verses[verse_tag][verse_num]) + 1
continue
# number at start of line.. it's verse number
if this_line[0].isdigit():
verse_num = this_line[0]
this_line = this_line[1:].strip()
verses.setdefault(verse_tag, {})
verses[verse_tag].setdefault(verse_num, {})
if inst not in verses[verse_tag][verse_num]:
verses[verse_tag][verse_num][inst] = []
our_verse_order.append([verse_tag, verse_num, inst])
# Tidy text and remove the ____s from extended words
this_line = self.tidy_text(this_line)
this_line = this_line.replace('_', '')
this_line = this_line.replace('||', '\n[---]\n')
this_line = this_line.strip()
# If the line consists solely of a '|', then just use the implicit newline
# Otherwise, add a newline for each '|'
if this_line == '|':
this_line = ''
else:
this_line = this_line.replace('|', '\n')
verses[verse_tag][verse_num][inst].append(this_line)
# done parsing
# add verses in original order
verse_joints = {}
for (verse_tag, verse_num, inst) in our_verse_order:
lines = '\n'.join(verses[verse_tag][verse_num][inst])
length = 0
while length < len(verse_num) and verse_num[length].isnumeric():
length += 1
verse_def = '%s%s' % (verse_tag, verse_num[:length])
verse_joints[verse_def] = '%s\n[---]\n%s' % (verse_joints[verse_def], lines) \
if verse_def in verse_joints else lines
# Parsing the dictionary produces the elements in a non-intuitive order. While it "works", it's not a
# natural layout should the user come back to edit the song. Instead we sort by the verse type, so that we
# get all the verses in order (v1, v2, ...), then the chorus(es), bridge(s), pre-chorus(es) etc. We use a
# tuple for the key, since tuples naturally sort in this manner.
verse_defs = sorted(verse_joints.keys(),
key=lambda verse_def: (VerseType.from_tag(verse_def[0]), int(verse_def[1:])))
for verse_def in verse_defs:
lines = verse_joints[verse_def]
self.add_verse(lines, verse_def)
if not self.verses:
self.add_verse('')
# figure out the presentation order, if present
if 'presentation' in fields and root.presentation:
order = str(root.presentation)
# We make all the tags in the lyrics lower case, so match that here and then split into a list on the
# whitespace.
order = order.lower().split()
for verse_def in order:
match = re.match('(\D*)(\d+.*)', verse_def)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
if not verse_tag:
verse_tag = VerseType.tags[VerseType.Verse]
else:
# Assume it's no.1 if there are no digits
verse_tag = verse_def
verse_num = '1'
verse_index = VerseType.from_loose_input(verse_tag)
verse_tag = VerseType.tags[verse_index]
verse_def = '%s%s' % (verse_tag, verse_num)
if verse_num in verses.get(verse_tag, {}):
self.verse_order_list.append(verse_def)
else:
log.info('Got order %s but not in verse tags, dropping this item from presentation order',
verse_def)
if not self.finish():
self.log_error(file.name)
def parse(self, data, cell=False):
"""
Process the records
:param data: The data to be processed
:param cell: ?
:return:
"""
if len(data) == 0 or data[0:1] != '[' or data[-1] != ']':
self.log_error('File is malformed')
return False
i = 1
verse_type = VerseType.tags[VerseType.Verse]
while i < len(data):
# Data is held as #name: value pairs inside groups marked as [].
# Now we are looking for the name.
if data[i:i + 1] == '#':
name_end = data.find(':', i + 1)
name = data[i + 1:name_end].upper()
i = name_end + 1
while data[i:i + 1] == ' ':
i += 1
if data[i:i + 1] == '"':
end = data.find('"', i + 1)
value = data[i + 1:end]
elif data[i:i + 1] == '[':
j = i
inside_quotes = False
while j < len(data):
char = data[j:j + 1]
if char == '"':
inside_quotes = not inside_quotes
elif not inside_quotes and char == ']':
end = j + 1
break
j += 1
value = data[i:end]
else:
end = data.find(',', i + 1)
if data.find('(', i, end) != -1:
end = data.find(')', i) + 1
value = data[i:end]
# If we are in the main group.
if not cell:
if name == 'TITLE':
self.title = self.decode(self.unescape(value))
elif name == 'AUTHOR':
author = self.decode(self.unescape(value))
if len(author):
self.add_author(author)
elif name == 'COPYRIGHT':
self.copyright = self.decode(self.unescape(value))
elif name[0:4] == 'CELL':
self.parse(value, cell=name[4:])
# We are in a verse group.
else:
if name == 'MARKER_NAME':
value = value.strip()
if len(value):
verse_type = VerseType.tags[VerseType.from_loose_input(value[0])]
if len(value) >= 2 and value[-1] in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']:
verse_type = "%s%s" % (verse_type, value[-1])
elif name == 'HOTKEY':
# HOTKEY always appears after MARKER_NAME, so it
# effectively overrides MARKER_NAME, if present.
if len(value) and value in list(HOTKEY_TO_VERSE_TYPE.keys()):
verse_type = HOTKEY_TO_VERSE_TYPE[value]
if name == 'RTF':
value = self.unescape(value)
result = strip_rtf(value, self.encoding)
if result is None:
return
verse, self.encoding = result
lines = verse.strip().split('\n')
# If any line inside any verse contains CCLI or
# only Public Domain, we treat this as special data:
# we remove that line and add data to specific field.
processed_lines = []
for i in range(len(lines)):
line = lines[i].strip()
if line[:3].lower() == 'ccl':
m = re.search(r'[0-9]+', line)
if m:
self.ccli_number = int(m.group(0))
continue
elif line.lower() == 'public domain':
self.copyright = 'Public Domain'
continue
processed_lines.append(line)
self.add_verse('\n'.join(processed_lines).strip(), verse_type)
if end == -1:
break
i = end + 1
i += 1
return True
def doImportFile(self, file):
"""
Process the OpenSong file - pass in a file-like object, not a file path.
"""
self.setDefaults()
try:
tree = objectify.parse(file)
except (Error, LxmlError):
self.logError(file.name, SongStrings.XMLSyntaxError)
log.exception(u'Error parsing XML')
return
root = tree.getroot()
if root.tag != u'song':
self.logError(file.name, unicode(
translate('SongsPlugin.OpenSongImport', ('Invalid OpenSong song file. Missing song tag.'))))
return
fields = dir(root)
decode = {
u'copyright': self.addCopyright,
u'ccli': u'ccli_number',
u'author': self.parseAuthor,
u'title': u'title',
u'aka': u'alternate_title',
u'hymn_number': u'song_number'
}
for attr, fn_or_string in decode.items():
if attr in fields:
ustring = unicode(root.__getattr__(attr))
if isinstance(fn_or_string, basestring):
setattr(self, fn_or_string, ustring)
else:
fn_or_string(ustring)
if u'theme' in fields and unicode(root.theme) not in self.topics:
self.topics.append(unicode(root.theme))
if u'alttheme' in fields and unicode(root.alttheme) not in self.topics:
self.topics.append(unicode(root.alttheme))
# data storage while importing
verses = {}
# keep track of verses appearance order
our_verse_order = []
# default verse
verse_tag = VerseType.Tags[VerseType.Verse]
verse_num = u'1'
# for the case where song has several sections with same marker
inst = 1
if u'lyrics' in fields:
lyrics = unicode(root.lyrics)
else:
lyrics = u''
for this_line in lyrics.split(u'\n'):
# remove comments
semicolon = this_line.find(u';')
if semicolon >= 0:
this_line = this_line[:semicolon]
this_line = this_line.strip()
if not this_line:
continue
# skip guitar chords and page and column breaks
if this_line.startswith(u'.') or this_line.startswith(u'---') or this_line.startswith(u'-!!'):
continue
# verse/chorus/etc. marker
if this_line.startswith(u'['):
# drop the square brackets
right_bracket = this_line.find(u']')
content = this_line[1:right_bracket].lower()
# have we got any digits?
# If so, verse number is everything from the digits
# to the end (openlp does not have concept of part verses, so
# just ignore any non integers on the end (including floats))
match = re.match(u'(\D*)(\d+)', content)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
else:
# otherwise we assume number 1 and take the whole prefix as
# the verse tag
verse_tag = content
verse_num = u'1'
verse_index = VerseType.from_loose_input(verse_tag) if verse_tag else 0
verse_tag = VerseType.Tags[verse_index]
inst = 1
if [verse_tag, verse_num, inst] in our_verse_order and verse_num in verses.get(verse_tag, {}):
inst = len(verses[verse_tag][verse_num]) + 1
continue
# number at start of line.. it's verse number
if this_line[0].isdigit():
verse_num = this_line[0]
this_line = this_line[1:].strip()
our_verse_order.append([verse_tag, verse_num, inst])
verses.setdefault(verse_tag, {})
verses[verse_tag].setdefault(verse_num, {})
if inst not in verses[verse_tag][verse_num]:
verses[verse_tag][verse_num][inst] = []
our_verse_order.append([verse_tag, verse_num, inst])
# Tidy text and remove the ____s from extended words
this_line = self.tidyText(this_line)
this_line = this_line.replace(u'_', u'')
this_line = this_line.replace(u'|', u'\n')
verses[verse_tag][verse_num][inst].append(this_line)
# done parsing
# add verses in original order
verse_joints = {}
for (verse_tag, verse_num, inst) in our_verse_order:
lines = u'\n'.join(verses[verse_tag][verse_num][inst])
length = 0
while(length < len(verse_num) and verse_num[length].isnumeric()):
length += 1
verse_def = u'%s%s' % (verse_tag, verse_num[:length])
verse_joints[verse_def] = u'%s\n[---]\n%s' % (verse_joints[verse_def], lines) \
if verse_def in verse_joints else lines
for verse_def, lines in verse_joints.iteritems():
self.addVerse(lines, verse_def)
if not self.verses:
self.addVerse('')
# figure out the presentation order, if present
if u'presentation' in fields and root.presentation:
order = unicode(root.presentation)
# We make all the tags in the lyrics lower case, so match that here
# and then split into a list on the whitespace
order = order.lower().split()
for verse_def in order:
match = re.match(u'(\D*)(\d+.*)', verse_def)
if match is not None:
verse_tag = match.group(1)
verse_num = match.group(2)
if not verse_tag:
verse_tag = VerseType.Tags[VerseType.Verse]
else:
# Assume it's no.1 if there are no digits
verse_tag = verse_def
verse_num = u'1'
verse_def = u'%s%s' % (verse_tag, verse_num)
if verse_num in verses.get(verse_tag, {}):
self.verseOrderList.append(verse_def)
else:
log.info(u'Got order %s but not in verse tags, dropping'
u'this item from presentation order', verse_def)
if not self.finish():
self.logError(file.name)
