def save_song(self, song): """ Save a song to the database, using the db_manager :param song: :return: """ db_song = Song.populate(title=song['title'], copyright=song['copyright'], ccli_number=song['ccli_number']) song_xml = SongXML() verse_order = [] for verse in song['verses']: verse_type, verse_number = verse['label'].split(' ')[:2] verse_type = VerseType.from_loose_input(verse_type) verse_number = int(verse_number) song_xml.add_verse_to_lyrics(VerseType.tags[verse_type], verse_number, verse['lyrics']) verse_order.append('%s%s' % (VerseType.tags[verse_type], verse_number)) db_song.verse_order = ' '.join(verse_order) db_song.lyrics = song_xml.extract_xml() clean_song(self.db_manager, db_song) self.db_manager.save_object(db_song) db_song.authors_songs = [] for author_name in song['authors']: author = self.db_manager.get_object_filtered(Author, Author.display_name == author_name) if not author: author = Author.populate(first_name=author_name.rsplit(' ', 1)[0], last_name=author_name.rsplit(' ', 1)[1], display_name=author_name) db_song.add_author(author) self.db_manager.save_object(db_song) return db_song
def on_verse_edit_all_button_clicked(self): verse_list = '' if self.verse_list_widget.rowCount() > 0: for row in range(self.verse_list_widget.rowCount()): item = self.verse_list_widget.item(row, 0) field = item.data(QtCore.Qt.UserRole) verse_tag = VerseType.translated_name(field[0]) verse_num = field[1:] verse_list += '---[%s:%s]---\n' % (verse_tag, verse_num) verse_list += item.text() verse_list += '\n' self.verse_form.set_verse(verse_list) else: self.verse_form.set_verse('') if not self.verse_form.exec_(): return verse_list = self.verse_form.get_all_verses() verse_list = str(verse_list.replace('\r\n', '\n')) self.verse_list_widget.clear() self.verse_list_widget.setRowCount(0) for row in self.find_verse_split.split(verse_list): for match in row.split('---['): for count, parts in enumerate(match.split(']---\n')): if count == 0: if len(parts) == 0: continue # handling carefully user inputted versetags separator = parts.find(':') if separator >= 0: verse_name = parts[0:separator].strip() verse_num = parts[separator+1:].strip() else: verse_name = parts verse_num = '1' verse_index = VerseType.from_loose_input(verse_name) verse_tag = VerseType.tags[verse_index] # Later we need to handle v1a as well. #regex = re.compile(r'(\d+\w.)') regex = re.compile(r'\D*(\d+)\D*') match = regex.match(verse_num) if match: verse_num = match.group(1) else: verse_num = '1' verse_def = '%s%s' % (verse_tag, verse_num) else: if parts.endswith('\n'): parts = parts.rstrip('\n') item = QtGui.QTableWidgetItem(parts) item.setData(QtCore.Qt.UserRole, verse_def) self.verse_list_widget.setRowCount(self.verse_list_widget.rowCount() + 1) self.verse_list_widget.setItem(self.verse_list_widget.rowCount() - 1, 0, item) self.tag_rows() self.verse_edit_button.setEnabled(False) self.verse_delete_button.setEnabled(False) # Check if all verse tags are used. self.on_verse_order_text_changed(self.verse_order_edit.text())
def test_from_loose_input_with_valid_input(self, mocked_translated_tags): """ Test that the from_loose_input() method returns valid output on valid input. """ # GIVEN: A mocked VerseType.translated_tags # WHEN: We run the from_loose_input() method with a valid verse type, we get the expected VerseType back result = VerseType.from_loose_input('v') # THEN: The result should be a Verse self.assertEqual(result, VerseType.Verse, 'The result should be a verse, but was "%s"' % result)
def test_from_loose_input_with_invalid_input(self, mocked_translated_tags): """ Test that the from_loose_input() method returns a sane default when passed an invalid tag and None as default. """ # GIVEN: A mocked VerseType.translated_tags # WHEN: We run the from_loose_input() method with an invalid verse type, we get the specified default back result = VerseType.from_loose_input('m', None) # THEN: The result should be None self.assertIsNone(result, 'The result should be None, but was "%s"' % result)
def test_from_loose_input_with_valid_input(self, mocked_translated_tags): """ Test that the from_loose_input() method returns valid output on valid input. """ # GIVEN: A mocked VerseType.translated_tags # WHEN: We run the from_loose_input() method with a valid verse type, we get the expected VerseType back result = VerseType.from_loose_input('v') # THEN: The result should be a Verse assert result == VerseType.Verse, 'The result should be a verse, but was "%s"' % result
def test_from_loose_input_with_invalid_input(self, mocked_translated_tags): """ Test that the from_loose_input() method returns a sane default when passed an invalid tag and None as default. """ # GIVEN: A mocked VerseType.translated_tags # WHEN: We run the from_loose_input() method with an invalid verse type, we get the specified default back result = VerseType.from_loose_input('m', None) # THEN: The result should be None assert result is None, 'The result should be None, but was "%s"' % result
def save_song(self, song): """ Save a song to the database, using the db_manager :param song: :return: """ db_song = Song.populate(title=song['title'], copyright=song['copyright'], ccli_number=song['ccli_number']) song_xml = SongXML() verse_order = [] for verse in song['verses']: if ' ' in verse['label']: verse_type, verse_number = verse['label'].split(' ', 1) else: verse_type = verse['label'] verse_number = 1 verse_type = VerseType.from_loose_input(verse_type) verse_number = int(verse_number) song_xml.add_verse_to_lyrics(VerseType.tags[verse_type], verse_number, verse['lyrics']) verse_order.append('%s%s' % (VerseType.tags[verse_type], verse_number)) db_song.verse_order = ' '.join(verse_order) db_song.lyrics = song_xml.extract_xml() clean_song(self.db_manager, db_song) self.db_manager.save_object(db_song) db_song.authors_songs = [] for author_name in song['authors']: author = self.db_manager.get_object_filtered( Author, Author.display_name == author_name) if not author: name_parts = author_name.rsplit(' ', 1) first_name = name_parts[0] if len(name_parts) == 1: last_name = '' else: last_name = name_parts[1] author = Author.populate(first_name=first_name, last_name=last_name, display_name=author_name) db_song.add_author(author) db_song.topics = [] for topic_name in song.get('topics', []): topic = self.db_manager.get_object_filtered( Topic, Topic.name == topic_name) if not topic: topic = Topic.populate(name=topic_name) db_song.topics.append(topic) self.db_manager.save_object(db_song) return db_song
def save_song(self, song): """ Save a song to the database, using the db_manager :param song: :return: """ db_song = Song.populate(title=song['title'], copyright=song['copyright'], ccli_number=song['ccli_number']) song_xml = SongXML() verse_order = [] for verse in song['verses']: if ' ' in verse['label']: verse_type, verse_number = verse['label'].split(' ', 1) else: verse_type = verse['label'] verse_number = 1 verse_type = VerseType.from_loose_input(verse_type) verse_number = int(verse_number) song_xml.add_verse_to_lyrics(VerseType.tags[verse_type], verse_number, verse['lyrics']) verse_order.append('{tag}{number}'.format(tag=VerseType.tags[verse_type], number=verse_number)) db_song.verse_order = ' '.join(verse_order) db_song.lyrics = song_xml.extract_xml() clean_song(self.db_manager, db_song) self.db_manager.save_object(db_song) db_song.authors_songs = [] for author_name in song['authors']: author = self.db_manager.get_object_filtered(Author, Author.display_name == author_name) if not author: name_parts = author_name.rsplit(' ', 1) first_name = name_parts[0] if len(name_parts) == 1: last_name = '' else: last_name = name_parts[1] author = Author.populate(first_name=first_name, last_name=last_name, display_name=author_name) db_song.add_author(author) for topic_name in song.get('topics', []): topic = self.db_manager.get_object_filtered(Topic, Topic.name == topic_name) if not topic: topic = Topic.populate(name=topic_name) db_song.topics.append(topic) self.db_manager.save_object(db_song) return db_song
def save_song(self, song): """ Save a song to the database, using the db_manager :param song: :return: """ db_song = Song.populate(title=song['title'], copyright=song['copyright'], ccli_number=song['ccli_number']) song_xml = SongXML() verse_order = [] for verse in song['verses']: verse_type, verse_number = verse['label'].split(' ')[:2] verse_type = VerseType.from_loose_input(verse_type) verse_number = int(verse_number) song_xml.add_verse_to_lyrics(VerseType.tags[verse_type], verse_number, verse['lyrics']) verse_order.append('%s%s' % (VerseType.tags[verse_type], verse_number)) db_song.verse_order = ' '.join(verse_order) db_song.lyrics = song_xml.extract_xml() clean_song(self.db_manager, db_song) self.db_manager.save_object(db_song) db_song.authors_songs = [] for author_name in song['authors']: author = self.db_manager.get_object_filtered( Author, Author.display_name == author_name) if not author: author = Author.populate(first_name=author_name.rsplit(' ', 1)[0], last_name=author_name.rsplit(' ', 1)[1], display_name=author_name) db_song.add_author(author) self.db_manager.save_object(db_song) return db_song
def do_import(self): """ Receive a CSV file to import. """ # Get encoding encoding = get_file_encoding(self.import_source)['encoding'] with self.import_source.open('r', encoding=encoding) as songs_file: songs_reader = csv.DictReader(songs_file, escapechar='\\') try: records = list(songs_reader) except csv.Error as e: self.log_error( translate('SongsPlugin.WorshipAssistantImport', 'Error reading CSV file.'), translate('SongsPlugin.WorshipAssistantImport', 'Line {number:d}: {error}').format( number=songs_reader.line_num, error=e)) return num_records = len(records) log.info('{count} records found in CSV file'.format(count=num_records)) self.import_wizard.progress_bar.setMaximum(num_records) # Create regex to strip html tags re_html_strip = re.compile(r'<[^>]+>') for index, record in enumerate(records, 1): if self.stop_import_flag: return # Ensure that all keys are uppercase record = dict( (field.upper(), value) for field, value in record.items()) # The CSV file has a line in the middle of the file where the headers are repeated. # We need to skip this line. if record['TITLE'] == "TITLE" and record[ 'AUTHOR'] == 'AUTHOR' and record['LYRICS2'] == 'LYRICS2': continue self.set_defaults() verse_order_list = [] try: self.title = record['TITLE'] if record['AUTHOR'] != EMPTY_STR: self.parse_author(record['AUTHOR']) if record['COPYRIGHT'] != EMPTY_STR: self.add_copyright(record['COPYRIGHT']) if record['CCLINR'] != EMPTY_STR: self.ccli_number = record['CCLINR'] if record['ROADMAP'] != EMPTY_STR: verse_order_list = [ x.strip() for x in record['ROADMAP'].split(',') ] lyrics = record['LYRICS2'] except UnicodeDecodeError as e: self.log_error( translate('SongsPlugin.WorshipAssistantImport', 'Record {count:d}').format(count=index), translate('SongsPlugin.WorshipAssistantImport', 'Decoding error: {error}').format(error=e)) continue except TypeError as e: self.log_error( translate('SongsPlugin.WorshipAssistantImport', 'File not valid WorshipAssistant CSV format.'), 'TypeError: {error}'.format(error=e)) return verse = '' used_verses = [] verse_id = VerseType.tags[VerseType.Verse] + '1' for line in lyrics.splitlines(): if line.startswith('['): # verse marker # Add previous verse if verse: # remove trailing linebreak, part of the WA syntax self.add_verse(verse[:-1], verse_id) used_verses.append(verse_id) verse = '' # drop the square brackets right_bracket = line.find(']') content = line[1:right_bracket].lower() match = re.match(r'(\D*)(\d+)', content) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) else: # otherwise we assume number 1 and take the whole prefix as the verse tag verse_tag = content verse_num = '1' verse_index = VerseType.from_loose_input( verse_tag) if verse_tag else 0 verse_tag = VerseType.tags[verse_index] # Update verse order when the verse name has changed verse_id = verse_tag + verse_num # Make sure we've not choosen an id already used while verse_id in verse_order_list and content in verse_order_list: verse_num = str(int(verse_num) + 1) verse_id = verse_tag + verse_num if content != verse_id: for i in range(len(verse_order_list)): if verse_order_list[i].lower() == content.lower(): verse_order_list[i] = verse_id else: # add line text to verse. Strip out html verse += re_html_strip.sub('', line) + '\n' if verse: # remove trailing linebreak, part of the WA syntax if verse.endswith('\n\n'): verse = verse[:-1] self.add_verse(verse, verse_id) used_verses.append(verse_id) if verse_order_list: # Use the verse order in the import, but remove entries that doesn't have a text cleaned_verse_order_list = [] for verse in verse_order_list: if verse in used_verses: cleaned_verse_order_list.append(verse) self.verse_order_list = cleaned_verse_order_list if not self.finish(): self.log_error( translate('SongsPlugin.WorshipAssistantImport', 'Record {count:d}').format(count=index) + (': "' + self.title + '"' if self.title else ''))
def do_import_file(self, file): """ Process the OpenSong file - pass in a file-like object, not a file path. """ self.set_defaults() try: tree = objectify.parse(file) except (Error, LxmlError): self.log_error(file.name, SongStrings.XMLSyntaxError) log.exception('Error parsing XML') return root = tree.getroot() if root.tag != 'song': self.log_error( file.name, str( translate( 'SongsPlugin.OpenSongImport', 'Invalid OpenSong song file. Missing song tag.'))) return fields = dir(root) decode = { 'copyright': self.add_copyright, 'ccli': 'ccli_number', 'author': self.parse_author, 'title': 'title', 'aka': 'alternate_title', 'hymn_number': self.parse_song_book_name_and_number, 'user1': self.add_comment, 'user2': self.add_comment, 'user3': self.add_comment } for attr, fn_or_string in list(decode.items()): if attr in fields: ustring = str(root.__getattr__(attr)) if isinstance(fn_or_string, str): if attr in ['ccli']: ustring = ''.join(re.findall('\d+', ustring)) if ustring: setattr(self, fn_or_string, int(ustring)) else: setattr(self, fn_or_string, None) else: setattr(self, fn_or_string, ustring) else: fn_or_string(ustring) # Themes look like "God: Awe/Wonder", but we just want # "Awe" and "Wonder". We use a set to ensure each topic # is only added once, in case it is already there, which # is actually quite likely if the alttheme is set topics = set(self.topics) if 'theme' in fields: theme = str(root.theme) subthemes = theme[theme.find(':') + 1:].split('/') for topic in subthemes: topics.add(topic.strip()) if 'alttheme' in fields: theme = str(root.alttheme) subthemes = theme[theme.find(':') + 1:].split('/') for topic in subthemes: topics.add(topic.strip()) self.topics = list(topics) self.topics.sort() # data storage while importing verses = {} # keep track of verses appearance order our_verse_order = [] # default verse verse_tag = VerseType.tags[VerseType.Verse] verse_num = '1' # for the case where song has several sections with same marker inst = 1 if 'lyrics' in fields: lyrics = str(root.lyrics) else: lyrics = '' for this_line in lyrics.split('\n'): if not this_line.strip(): continue # skip this line if it is a comment if this_line.startswith(';'): continue # skip guitar chords and page and column breaks if this_line.startswith('.') or this_line.startswith( '---') or this_line.startswith('-!!'): continue # verse/chorus/etc. marker if this_line.startswith('['): # drop the square brackets right_bracket = this_line.find(']') content = this_line[1:right_bracket].lower() # have we got any digits? If so, verse number is everything from the digits to the end (openlp does not # have concept of part verses, so just ignore any non integers on the end (including floats)) match = re.match('(\D*)(\d+)', content) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) else: # otherwise we assume number 1 and take the whole prefix as the verse tag verse_tag = content verse_num = '1' verse_index = VerseType.from_loose_input( verse_tag) if verse_tag else 0 verse_tag = VerseType.tags[verse_index] inst = 1 if [verse_tag, verse_num, inst ] in our_verse_order and verse_num in verses.get( verse_tag, {}): inst = len(verses[verse_tag][verse_num]) + 1 continue # number at start of line.. it's verse number if this_line[0].isdigit(): verse_num = this_line[0] this_line = this_line[1:].strip() verses.setdefault(verse_tag, {}) verses[verse_tag].setdefault(verse_num, {}) if inst not in verses[verse_tag][verse_num]: verses[verse_tag][verse_num][inst] = [] our_verse_order.append([verse_tag, verse_num, inst]) # Tidy text and remove the ____s from extended words this_line = self.tidy_text(this_line) this_line = this_line.replace('_', '') this_line = this_line.replace('||', '\n[---]\n') this_line = this_line.strip() # If the line consists solely of a '|', then just use the implicit newline # Otherwise, add a newline for each '|' if this_line == '|': this_line = '' else: this_line = this_line.replace('|', '\n') verses[verse_tag][verse_num][inst].append(this_line) # done parsing # add verses in original order verse_joints = {} for (verse_tag, verse_num, inst) in our_verse_order: lines = '\n'.join(verses[verse_tag][verse_num][inst]) length = 0 while length < len(verse_num) and verse_num[length].isnumeric(): length += 1 verse_def = '{tag}{number}'.format(tag=verse_tag, number=verse_num[:length]) verse_joints[verse_def] = '{verse}\n[---]\n{lines}'.format(verse=verse_joints[verse_def], lines=lines) \ if verse_def in verse_joints else lines # Parsing the dictionary produces the elements in a non-intuitive order. While it "works", it's not a # natural layout should the user come back to edit the song. Instead we sort by the verse type, so that we # get all the verses in order (v1, v2, ...), then the chorus(es), bridge(s), pre-chorus(es) etc. We use a # tuple for the key, since tuples naturally sort in this manner. verse_defs = sorted( verse_joints.keys(), key=lambda verse_def: (VerseType.from_tag(verse_def[0]), int(verse_def[1:]))) for verse_def in verse_defs: lines = verse_joints[verse_def] self.add_verse(lines, verse_def) if not self.verses: self.add_verse('') # figure out the presentation order, if present if 'presentation' in fields and root.presentation: order = str(root.presentation) # We make all the tags in the lyrics lower case, so match that here and then split into a list on the # whitespace. order = order.lower().split() for verse_def in order: match = re.match('(\D*)(\d+.*)', verse_def) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) if not verse_tag: verse_tag = VerseType.tags[VerseType.Verse] else: # Assume it's no.1 if there are no digits verse_tag = verse_def verse_num = '1' verse_index = VerseType.from_loose_input(verse_tag) verse_tag = VerseType.tags[verse_index] verse_def = '{tag}{number}'.format(tag=verse_tag, number=verse_num) if verse_num in verses.get(verse_tag, {}): self.verse_order_list.append(verse_def) else: log.info( 'Got order {order} but not in verse tags, dropping this item from presentation ' 'order'.format(order=verse_def)) if not self.finish(): self.log_error(file.name)
def do_import(self): """ Receive a CSV file to import. """ # Get encoding detect_file = open(self.import_source, 'rb') detect_content = detect_file.read() details = chardet.detect(detect_content) detect_file.close() songs_file = open(self.import_source, 'r', encoding=details['encoding']) songs_reader = csv.DictReader(songs_file, escapechar='\\') try: records = list(songs_reader) except csv.Error as e: self.log_error(translate('SongsPlugin.WorshipAssistantImport', 'Error reading CSV file.'), translate('SongsPlugin.WorshipAssistantImport', 'Line %d: %s') % (songs_reader.line_num, e)) return num_records = len(records) log.info('%s records found in CSV file' % num_records) self.import_wizard.progress_bar.setMaximum(num_records) # Create regex to strip html tags re_html_strip = re.compile(r'<[^>]+>') for index, record in enumerate(records, 1): if self.stop_import_flag: return # Ensure that all keys are uppercase record = dict((field.upper(), value) for field, value in record.items()) # The CSV file has a line in the middle of the file where the headers are repeated. # We need to skip this line. if record['TITLE'] == "TITLE" and record['AUTHOR'] == 'AUTHOR' and record['LYRICS2'] == 'LYRICS2': continue self.set_defaults() verse_order_list = [] try: self.title = record['TITLE'] if record['AUTHOR'] != EMPTY_STR: self.parse_author(record['AUTHOR']) if record['COPYRIGHT'] != EMPTY_STR: self.add_copyright(record['COPYRIGHT']) if record['CCLINR'] != EMPTY_STR: self.ccli_number = record['CCLINR'] if record['ROADMAP'] != EMPTY_STR: verse_order_list = [x.strip() for x in record['ROADMAP'].split(',')] lyrics = record['LYRICS2'] except UnicodeDecodeError as e: self.log_error(translate('SongsPlugin.WorshipAssistantImport', 'Record %d' % index), translate('SongsPlugin.WorshipAssistantImport', 'Decoding error: %s') % e) continue except TypeError as e: self.log_error(translate('SongsPlugin.WorshipAssistantImport', 'File not valid WorshipAssistant CSV format.'), 'TypeError: %s' % e) return verse = '' used_verses = [] verse_id = VerseType.tags[VerseType.Verse] + '1' for line in lyrics.splitlines(): if line.startswith('['): # verse marker # Add previous verse if verse: # remove trailing linebreak, part of the WA syntax self.add_verse(verse[:-1], verse_id) used_verses.append(verse_id) verse = '' # drop the square brackets right_bracket = line.find(']') content = line[1:right_bracket].lower() match = re.match('(\D*)(\d+)', content) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) else: # otherwise we assume number 1 and take the whole prefix as the verse tag verse_tag = content verse_num = '1' verse_index = VerseType.from_loose_input(verse_tag) if verse_tag else 0 verse_tag = VerseType.tags[verse_index] # Update verse order when the verse name has changed verse_id = verse_tag + verse_num # Make sure we've not choosen an id already used while verse_id in verse_order_list and content in verse_order_list: verse_num = str(int(verse_num) + 1) verse_id = verse_tag + verse_num if content != verse_id: for i in range(len(verse_order_list)): if verse_order_list[i].lower() == content.lower(): verse_order_list[i] = verse_id else: # add line text to verse. Strip out html verse += re_html_strip.sub('', line) + '\n' if verse: # remove trailing linebreak, part of the WA syntax if verse.endswith('\n\n'): verse = verse[:-1] self.add_verse(verse, verse_id) used_verses.append(verse_id) if verse_order_list: # Use the verse order in the import, but remove entries that doesn't have a text cleaned_verse_order_list = [] for verse in verse_order_list: if verse in used_verses: cleaned_verse_order_list.append(verse) self.verse_order_list = cleaned_verse_order_list if not self.finish(): self.log_error(translate('SongsPlugin.WorshipAssistantImport', 'Record %d') % index + (': "' + self.title + '"' if self.title else '')) songs_file.close()
def parse(self, data, cell=False): """ Process the records :param data: The data to be processed :param cell: ? :return: """ if not cell and (len(data) == 0 or data[0:1] != b'[' or data.strip()[-1:] != b']'): self.log_error('File is malformed') return False i = 1 verse_type = VerseType.tags[VerseType.Verse] while i < len(data): # Data is held as #name: value pairs inside groups marked as []. # Now we are looking for the name. if data[i:i + 1] == b'#': name_end = data.find(b':', i + 1) name = data[i + 1:name_end].decode(self.encoding).upper() i = name_end + 1 while data[i:i + 1] == b' ': i += 1 if data[i:i + 1] == b'"': end = data.find(b'"', i + 1) value = data[i + 1:end] elif data[i:i + 1] == b'[': j = i inside_quotes = False while j < len(data): char = data[j:j + 1] if char == b'"': inside_quotes = not inside_quotes elif not inside_quotes and char == b']': end = j + 1 break j += 1 value = data[i:end] else: end = data.find(b',', i + 1) if data.find(b'(', i, end) != -1: end = data.find(b')', i) + 1 value = data[i:end] # If we are in the main group. if not cell: if name == 'TITLE': self.title = self.decode(self.unescape(value)) elif name == 'AUTHOR': author = self.decode(self.unescape(value)) if len(author): self.add_author(author) elif name == 'COPYRIGHT': self.add_copyright(self.decode(self.unescape(value))) elif name[0:4] == 'CELL': self.parse(value, cell=name[4:]) # We are in a verse group. else: if name == 'MARKER_NAME': value = self.decode(value).strip() if len(value): verse_type = VerseType.tags[ VerseType.from_loose_input(value[0])] if len(value) >= 2 and value[-1] in [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' ]: verse_type = "{verse}{value}".format( verse=verse_type, value=value[-1]) elif name == 'HOTKEY': value = self.decode(value).strip() # HOTKEY always appears after MARKER_NAME, so it # effectively overrides MARKER_NAME, if present. if len(value) and value in list( HOTKEY_TO_VERSE_TYPE.keys()): verse_type = HOTKEY_TO_VERSE_TYPE[value] if name == 'RTF': value = self.unescape(value) value = self.decode(value) result = strip_rtf(value, self.encoding) if result is None: return False verse, self.encoding = result lines = verse.strip().split('\n') # If any line inside any verse contains CCLI or # only Public Domain, we treat this as special data: # we remove that line and add data to specific field. processed_lines = [] for i in range(len(lines)): line = lines[i].strip() if line[:3].lower() == 'ccl': m = re.search(r'[0-9]+', line) if m: self.ccli_number = int(m.group(0)) continue elif line.lower() == 'public domain': self.add_copyright('Public Domain') continue processed_lines.append(line) self.add_verse('\n'.join(processed_lines).strip(), verse_type) if end == -1: break i = end + 1 i += 1 return True
def do_import_file(self, file): """ Process the OpenSong file - pass in a file-like object, not a file path. """ self.set_defaults() try: tree = objectify.parse(file) except (Error, LxmlError): self.log_error(file.name, SongStrings.XMLSyntaxError) log.exception('Error parsing XML') return root = tree.getroot() if root.tag != 'song': self.log_error(file.name, str( translate('SongsPlugin.OpenSongImport', 'Invalid OpenSong song file. Missing song tag.'))) return fields = dir(root) decode = { 'copyright': self.add_copyright, 'ccli': 'ccli_number', 'author': self.parse_author, 'title': 'title', 'aka': 'alternate_title', 'hymn_number': self.parse_song_book_name_and_number, 'user1': self.add_comment, 'user2': self.add_comment, 'user3': self.add_comment } for attr, fn_or_string in list(decode.items()): if attr in fields: ustring = str(root.__getattr__(attr)) if isinstance(fn_or_string, str): if attr in ['ccli']: if ustring: setattr(self, fn_or_string, int(ustring)) else: setattr(self, fn_or_string, None) else: setattr(self, fn_or_string, ustring) else: fn_or_string(ustring) # Themes look like "God: Awe/Wonder", but we just want # "Awe" and "Wonder". We use a set to ensure each topic # is only added once, in case it is already there, which # is actually quite likely if the alttheme is set topics = set(self.topics) if 'theme' in fields: theme = str(root.theme) subthemes = theme[theme.find(':')+1:].split('/') for topic in subthemes: topics.add(topic.strip()) if 'alttheme' in fields: theme = str(root.alttheme) subthemes = theme[theme.find(':')+1:].split('/') for topic in subthemes: topics.add(topic.strip()) self.topics = list(topics) self.topics.sort() # data storage while importing verses = {} # keep track of verses appearance order our_verse_order = [] # default verse verse_tag = VerseType.tags[VerseType.Verse] verse_num = '1' # for the case where song has several sections with same marker inst = 1 if 'lyrics' in fields: lyrics = str(root.lyrics) else: lyrics = '' for this_line in lyrics.split('\n'): if not this_line.strip(): continue # skip this line if it is a comment if this_line.startswith(';'): continue # skip guitar chords and page and column breaks if this_line.startswith('.') or this_line.startswith('---') or this_line.startswith('-!!'): continue # verse/chorus/etc. marker if this_line.startswith('['): # drop the square brackets right_bracket = this_line.find(']') content = this_line[1:right_bracket].lower() # have we got any digits? If so, verse number is everything from the digits to the end (openlp does not # have concept of part verses, so just ignore any non integers on the end (including floats)) match = re.match('(\D*)(\d+)', content) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) else: # otherwise we assume number 1 and take the whole prefix as the verse tag verse_tag = content verse_num = '1' verse_index = VerseType.from_loose_input(verse_tag) if verse_tag else 0 verse_tag = VerseType.tags[verse_index] inst = 1 if [verse_tag, verse_num, inst] in our_verse_order and verse_num in verses.get(verse_tag, {}): inst = len(verses[verse_tag][verse_num]) + 1 continue # number at start of line.. it's verse number if this_line[0].isdigit(): verse_num = this_line[0] this_line = this_line[1:].strip() verses.setdefault(verse_tag, {}) verses[verse_tag].setdefault(verse_num, {}) if inst not in verses[verse_tag][verse_num]: verses[verse_tag][verse_num][inst] = [] our_verse_order.append([verse_tag, verse_num, inst]) # Tidy text and remove the ____s from extended words this_line = self.tidy_text(this_line) this_line = this_line.replace('_', '') this_line = this_line.replace('||', '\n[---]\n') this_line = this_line.strip() # If the line consists solely of a '|', then just use the implicit newline # Otherwise, add a newline for each '|' if this_line == '|': this_line = '' else: this_line = this_line.replace('|', '\n') verses[verse_tag][verse_num][inst].append(this_line) # done parsing # add verses in original order verse_joints = {} for (verse_tag, verse_num, inst) in our_verse_order: lines = '\n'.join(verses[verse_tag][verse_num][inst]) length = 0 while length < len(verse_num) and verse_num[length].isnumeric(): length += 1 verse_def = '%s%s' % (verse_tag, verse_num[:length]) verse_joints[verse_def] = '%s\n[---]\n%s' % (verse_joints[verse_def], lines) \ if verse_def in verse_joints else lines # Parsing the dictionary produces the elements in a non-intuitive order. While it "works", it's not a # natural layout should the user come back to edit the song. Instead we sort by the verse type, so that we # get all the verses in order (v1, v2, ...), then the chorus(es), bridge(s), pre-chorus(es) etc. We use a # tuple for the key, since tuples naturally sort in this manner. verse_defs = sorted(verse_joints.keys(), key=lambda verse_def: (VerseType.from_tag(verse_def[0]), int(verse_def[1:]))) for verse_def in verse_defs: lines = verse_joints[verse_def] self.add_verse(lines, verse_def) if not self.verses: self.add_verse('') # figure out the presentation order, if present if 'presentation' in fields and root.presentation: order = str(root.presentation) # We make all the tags in the lyrics lower case, so match that here and then split into a list on the # whitespace. order = order.lower().split() for verse_def in order: match = re.match('(\D*)(\d+.*)', verse_def) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) if not verse_tag: verse_tag = VerseType.tags[VerseType.Verse] else: # Assume it's no.1 if there are no digits verse_tag = verse_def verse_num = '1' verse_index = VerseType.from_loose_input(verse_tag) verse_tag = VerseType.tags[verse_index] verse_def = '%s%s' % (verse_tag, verse_num) if verse_num in verses.get(verse_tag, {}): self.verse_order_list.append(verse_def) else: log.info('Got order %s but not in verse tags, dropping this item from presentation order', verse_def) if not self.finish(): self.log_error(file.name)
def parse(self, data, cell=False): """ Process the records :param data: The data to be processed :param cell: ? :return: """ if len(data) == 0 or data[0:1] != '[' or data[-1] != ']': self.log_error('File is malformed') return False i = 1 verse_type = VerseType.tags[VerseType.Verse] while i < len(data): # Data is held as #name: value pairs inside groups marked as []. # Now we are looking for the name. if data[i:i + 1] == '#': name_end = data.find(':', i + 1) name = data[i + 1:name_end].upper() i = name_end + 1 while data[i:i + 1] == ' ': i += 1 if data[i:i + 1] == '"': end = data.find('"', i + 1) value = data[i + 1:end] elif data[i:i + 1] == '[': j = i inside_quotes = False while j < len(data): char = data[j:j + 1] if char == '"': inside_quotes = not inside_quotes elif not inside_quotes and char == ']': end = j + 1 break j += 1 value = data[i:end] else: end = data.find(',', i + 1) if data.find('(', i, end) != -1: end = data.find(')', i) + 1 value = data[i:end] # If we are in the main group. if not cell: if name == 'TITLE': self.title = self.decode(self.unescape(value)) elif name == 'AUTHOR': author = self.decode(self.unescape(value)) if len(author): self.add_author(author) elif name == 'COPYRIGHT': self.copyright = self.decode(self.unescape(value)) elif name[0:4] == 'CELL': self.parse(value, cell=name[4:]) # We are in a verse group. else: if name == 'MARKER_NAME': value = value.strip() if len(value): verse_type = VerseType.tags[VerseType.from_loose_input(value[0])] if len(value) >= 2 and value[-1] in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']: verse_type = "%s%s" % (verse_type, value[-1]) elif name == 'HOTKEY': # HOTKEY always appears after MARKER_NAME, so it # effectively overrides MARKER_NAME, if present. if len(value) and value in list(HOTKEY_TO_VERSE_TYPE.keys()): verse_type = HOTKEY_TO_VERSE_TYPE[value] if name == 'RTF': value = self.unescape(value) result = strip_rtf(value, self.encoding) if result is None: return verse, self.encoding = result lines = verse.strip().split('\n') # If any line inside any verse contains CCLI or # only Public Domain, we treat this as special data: # we remove that line and add data to specific field. processed_lines = [] for i in range(len(lines)): line = lines[i].strip() if line[:3].lower() == 'ccl': m = re.search(r'[0-9]+', line) if m: self.ccli_number = int(m.group(0)) continue elif line.lower() == 'public domain': self.copyright = 'Public Domain' continue processed_lines.append(line) self.add_verse('\n'.join(processed_lines).strip(), verse_type) if end == -1: break i = end + 1 i += 1 return True
def doImportFile(self, file): """ Process the OpenSong file - pass in a file-like object, not a file path. """ self.setDefaults() try: tree = objectify.parse(file) except (Error, LxmlError): self.logError(file.name, SongStrings.XMLSyntaxError) log.exception(u'Error parsing XML') return root = tree.getroot() if root.tag != u'song': self.logError(file.name, unicode( translate('SongsPlugin.OpenSongImport', ('Invalid OpenSong song file. Missing song tag.')))) return fields = dir(root) decode = { u'copyright': self.addCopyright, u'ccli': u'ccli_number', u'author': self.parseAuthor, u'title': u'title', u'aka': u'alternate_title', u'hymn_number': u'song_number' } for attr, fn_or_string in decode.items(): if attr in fields: ustring = unicode(root.__getattr__(attr)) if isinstance(fn_or_string, basestring): setattr(self, fn_or_string, ustring) else: fn_or_string(ustring) if u'theme' in fields and unicode(root.theme) not in self.topics: self.topics.append(unicode(root.theme)) if u'alttheme' in fields and unicode(root.alttheme) not in self.topics: self.topics.append(unicode(root.alttheme)) # data storage while importing verses = {} # keep track of verses appearance order our_verse_order = [] # default verse verse_tag = VerseType.Tags[VerseType.Verse] verse_num = u'1' # for the case where song has several sections with same marker inst = 1 if u'lyrics' in fields: lyrics = unicode(root.lyrics) else: lyrics = u'' for this_line in lyrics.split(u'\n'): # remove comments semicolon = this_line.find(u';') if semicolon >= 0: this_line = this_line[:semicolon] this_line = this_line.strip() if not this_line: continue # skip guitar chords and page and column breaks if this_line.startswith(u'.') or this_line.startswith(u'---') or this_line.startswith(u'-!!'): continue # verse/chorus/etc. marker if this_line.startswith(u'['): # drop the square brackets right_bracket = this_line.find(u']') content = this_line[1:right_bracket].lower() # have we got any digits? # If so, verse number is everything from the digits # to the end (openlp does not have concept of part verses, so # just ignore any non integers on the end (including floats)) match = re.match(u'(\D*)(\d+)', content) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) else: # otherwise we assume number 1 and take the whole prefix as # the verse tag verse_tag = content verse_num = u'1' verse_index = VerseType.from_loose_input(verse_tag) if verse_tag else 0 verse_tag = VerseType.Tags[verse_index] inst = 1 if [verse_tag, verse_num, inst] in our_verse_order and verse_num in verses.get(verse_tag, {}): inst = len(verses[verse_tag][verse_num]) + 1 continue # number at start of line.. it's verse number if this_line[0].isdigit(): verse_num = this_line[0] this_line = this_line[1:].strip() our_verse_order.append([verse_tag, verse_num, inst]) verses.setdefault(verse_tag, {}) verses[verse_tag].setdefault(verse_num, {}) if inst not in verses[verse_tag][verse_num]: verses[verse_tag][verse_num][inst] = [] our_verse_order.append([verse_tag, verse_num, inst]) # Tidy text and remove the ____s from extended words this_line = self.tidyText(this_line) this_line = this_line.replace(u'_', u'') this_line = this_line.replace(u'|', u'\n') verses[verse_tag][verse_num][inst].append(this_line) # done parsing # add verses in original order verse_joints = {} for (verse_tag, verse_num, inst) in our_verse_order: lines = u'\n'.join(verses[verse_tag][verse_num][inst]) length = 0 while(length < len(verse_num) and verse_num[length].isnumeric()): length += 1 verse_def = u'%s%s' % (verse_tag, verse_num[:length]) verse_joints[verse_def] = u'%s\n[---]\n%s' % (verse_joints[verse_def], lines) \ if verse_def in verse_joints else lines for verse_def, lines in verse_joints.iteritems(): self.addVerse(lines, verse_def) if not self.verses: self.addVerse('') # figure out the presentation order, if present if u'presentation' in fields and root.presentation: order = unicode(root.presentation) # We make all the tags in the lyrics lower case, so match that here # and then split into a list on the whitespace order = order.lower().split() for verse_def in order: match = re.match(u'(\D*)(\d+.*)', verse_def) if match is not None: verse_tag = match.group(1) verse_num = match.group(2) if not verse_tag: verse_tag = VerseType.Tags[VerseType.Verse] else: # Assume it's no.1 if there are no digits verse_tag = verse_def verse_num = u'1' verse_def = u'%s%s' % (verse_tag, verse_num) if verse_num in verses.get(verse_tag, {}): self.verseOrderList.append(verse_def) else: log.info(u'Got order %s but not in verse tags, dropping' u'this item from presentation order', verse_def) if not self.finish(): self.logError(file.name)