def execute(self):
logging.debug("Verse worker started!")
self.clear_report()
self.__temp_dir = init_temp_dir()
for src_file in self.__ftp_dir.rglob('*.wav'):
# Process verse/chunk files only
if not re.search(self.__verse_regex, str(src_file)):
continue
# Extract necessary path parts
root_parts = self.__ftp_dir.parts
parts = src_file.parts[len(root_parts):]
lang = parts[0]
resource = parts[1]
book = parts[2]
chapter = parts[3]
media = parts[5]
grouping = parts[7] if media == 'mp3' else parts[6]
target_dir = self.__temp_dir.joinpath(lang, resource, book,
chapter, grouping)
remote_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter,
"CONTENTS")
target_dir.mkdir(parents=True, exist_ok=True)
target_file = target_dir.joinpath(src_file.name)
logging.debug(f'Found verse file: {src_file}')
mp3_exists = check_file_exists(src_file, remote_dir, 'mp3',
grouping)
cue_exists = check_file_exists(src_file, remote_dir, 'cue',
grouping)
if mp3_exists and cue_exists:
logging.debug(f'Files exist. Skipping...')
continue
# Copy source file to temp dir
logging.debug(f'Copying file {src_file} to {target_file}')
target_file.write_bytes(src_file.read_bytes())
# Try to fix wav metadata
logging.debug(f'Fixing metadata: {target_file}')
fix_metadata(target_file, self.verbose)
# Convert verse into mp3
self.convert_verse(target_file, remote_dir, grouping)
logging.debug(f'Deleting temporary directory {self.__temp_dir}')
rm_tree(self.__temp_dir)
logging.debug('Verse worker finished!')
def delete_verse_files(self, lang, resource, book, chapter):
""" Delete verse related files (wav, mp3, cue) """
remote_chapter_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS")
wav_verse_files = remote_chapter_dir.joinpath("wav", "verse")
if wav_verse_files.exists():
rm_tree(wav_verse_files)
self.resources_deleted.append(str(rel_path(wav_verse_files, self.__ftp_dir)))
mp3_hi_verse_dir = remote_chapter_dir.joinpath("mp3", "hi", "verse")
if mp3_hi_verse_dir.exists():
rm_tree(mp3_hi_verse_dir)
self.resources_deleted.append(str(rel_path(mp3_hi_verse_dir, self.__ftp_dir)))
mp3_low_verse_dir = remote_chapter_dir.joinpath("mp3", "low", "verse")
if mp3_low_verse_dir.exists():
rm_tree(mp3_low_verse_dir)
self.resources_deleted.append(str(rel_path(mp3_low_verse_dir, self.__ftp_dir)))
cue_verse_dir = remote_chapter_dir.joinpath("cue", "verse")
if cue_verse_dir.exists():
rm_tree(cue_verse_dir)
self.resources_deleted.append(str(rel_path(cue_verse_dir, self.__ftp_dir)))
def delete_chapter_files(self, lang, resource, book, chapter):
""" Delete chapter related files (mp3 and cue) """
remote_chapter_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS")
mp3_hi_chapter_dir = remote_chapter_dir.joinpath("mp3", "hi", "chapter")
if mp3_hi_chapter_dir.exists():
rm_tree(mp3_hi_chapter_dir)
self.resources_deleted.append(str(rel_path(mp3_hi_chapter_dir, self.__ftp_dir)))
mp3_low_chapter_dir = remote_chapter_dir.joinpath("mp3", "low", "chapter")
if mp3_low_chapter_dir.exists():
rm_tree(mp3_low_chapter_dir)
self.resources_deleted.append(str(rel_path(mp3_low_chapter_dir, self.__ftp_dir)))
cue_chapter_dir = remote_chapter_dir.joinpath("cue", "chapter")
if cue_chapter_dir.exists():
rm_tree(cue_chapter_dir)
self.resources_deleted.append(str(rel_path(cue_chapter_dir, self.__ftp_dir)))
def execute(self):
""" Execute worker """
logging.debug("TR worker started!")
self.clear_report()
self.__temp_dir = init_temp_dir()
existent_tr = self.find_existent_tr()
media = ['wav', 'mp3/hi', 'mp3/low']
for m in media:
for src_file in self.__ftp_dir.rglob(f'{m}/verse/*.*'):
if src_file.suffix == '.tr':
continue
# Process verse files only
if not re.search(self.__verse_regex, str(src_file)):
continue
self.__book_tr_files.append(src_file)
self.__chapter_tr_files.append(src_file)
# Extract necessary path parts
root_parts = self.__ftp_dir.parts
parts = src_file.parts[len(root_parts):]
lang = parts[0]
resource = parts[1]
book = parts[2]
chapter = parts[3]
media = parts[5]
quality = parts[6] if media == 'mp3' else ''
grouping = parts[7] if media == 'mp3' else parts[6]
regex = fr'{lang}\/{resource}\/{book}(?:\/{chapter})?\/' \
fr'CONTENTS\/tr\/{media}(?:\/{quality})?\/{grouping}'
for group, tr in existent_tr:
if not re.search(regex, str(tr)):
continue
if group == Group.BOOK and src_file in self.__book_tr_files:
self.__book_tr_files.remove(src_file)
elif group == Group.CHAPTER and src_file in self.__chapter_tr_files:
self.__chapter_tr_files.remove(src_file)
# Create chapter TRs
chapter_groups = self.group_files(self.__chapter_tr_files,
Group.CHAPTER)
for key in chapter_groups:
self.create_tr_file(key, chapter_groups[key])
# Create book TRs
book_groups = self.group_files(self.__book_tr_files, Group.BOOK)
for key in book_groups:
self.create_tr_file(key, book_groups[key])
logging.debug(f'Deleting temporary directory {self.__temp_dir}')
rm_tree(self.__temp_dir)
logging.debug('TR worker finished!')
def create_tr_file(self, dic: str, files: List[Path]):
""" Create tr file and copy it to the remote directory"""
parts = json.loads(dic)
lang = parts['lang']
resource = parts['resource']
book = parts['book']
chapter = parts['chapter'] if 'chapter' in parts else None
media = parts['media']
quality = parts['quality']
grouping = parts['grouping']
root_dir = self.__temp_dir.joinpath('root')
target_dir = root_dir.joinpath(lang, resource, book)
if chapter is not None:
remote_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter,
"CONTENTS")
else:
remote_dir = self.__ftp_dir.joinpath(lang, resource, book,
"CONTENTS")
for file in files:
target_chapter = chapter
if target_chapter is None:
match = re.search(r'_c([0-9]+)_v[0-9]+', file.name)
if not match:
raise Exception(
'Could not define chapter from the file name.')
target_chapter = match.group(1)
target_chapter_dir = target_dir.joinpath(
self.zero_pad_chapter(target_chapter, book))
target_chapter_dir.mkdir(parents=True, exist_ok=True)
target_file = target_chapter_dir.joinpath(file.name)
# Copy source file to temp dir
logging.debug(f'Copying file {file} to {target_file}')
target_file.write_bytes(file.read_bytes())
# Create TR file
logging.debug('Creating TR file')
create_tr(root_dir, self.verbose)
tr = self.__temp_dir.joinpath('root.tr')
if chapter is not None:
new_tr = Path(tr.parent, f'{lang}_{resource}_{book}_c{chapter}.tr')
else:
new_tr = Path(tr.parent, f'{lang}_{resource}_{book}.tr')
tr.rename(new_tr)
# Copy tr file to remote dir
logging.debug(f'Copying {new_tr} to {remote_dir}')
t_file = copy_file(new_tr, remote_dir, grouping, quality, media)
self.resources_created.append(str(rel_path(t_file, self.__ftp_dir)))
rm_tree(root_dir)
new_tr.unlink()
def execute(self):
""" Execute worker """
logging.debug("Chapter worker started!")
self.clear_report()
self.__temp_dir = init_temp_dir()
for src_file in self.__ftp_dir.rglob('*.wav'):
# Process chapter files only
if not re.search(self.__chapter_regex, str(src_file)):
continue
# Extract necessary path parts
root_parts = self.__ftp_dir.parts
parts = src_file.parts[len(root_parts):]
lang = parts[0]
resource = parts[1]
book = parts[2]
chapter = parts[3]
grouping = parts[6]
target_dir = self.__temp_dir.joinpath(lang, resource, book, chapter, grouping)
remote_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS")
verses_dir = target_dir.joinpath("verses")
verses_dir.mkdir(parents=True, exist_ok=True)
target_file = target_dir.joinpath(src_file.name)
logging.debug(f'Found chapter file: {src_file}')
# Copy source file to temp dir
logging.debug(f'Copying file {src_file} to {target_file}')
target_file.write_bytes(src_file.read_bytes())
# Try to fix wav metadata
logging.debug(f'Fixing metadata: {target_file}')
fix_metadata(target_file, self.verbose)
# Split chapter files into verses
logging.debug(f'Splitting chapter {target_file} into {verses_dir}')
split_chapter(target_file, verses_dir, self.verbose)
target_verse_dir = remote_dir.joinpath("wav", "verse")
is_new = check_dir_empty(target_verse_dir)
is_changed = has_new_files(verses_dir, target_verse_dir)
cleaning = is_new or is_changed
# If we have a new or updated chapter WAV file
# delete all the chapter related resources:
# split verses, converted files and TR files
if cleaning:
# Delete related chapter files
self.delete_chapter_files(lang, resource, book, chapter)
# Delete related verse files
self.delete_verse_files(lang, resource, book, chapter)
# Delete related book TR files
self.delete_tr_book_files(lang, resource, book)
# Delete related chapter TR files
self.delete_tr_chapter_files(lang, resource, book, chapter)
# Copy original verse files
logging.debug(
f'Copying original verse files from {verses_dir} into {remote_dir}'
)
t_dir = copy_dir(verses_dir, remote_dir)
if cleaning:
self.resources_created.append(str(rel_path(t_dir, self.__ftp_dir)))
# Convert chapter into mp3
self.convert_chapter(target_file, remote_dir, grouping)
# Convert verses into mp3
self.convert_verses(verses_dir, remote_dir)
logging.debug(f'Deleting temporary directory {self.__temp_dir}')
rm_tree(self.__temp_dir)
logging.debug('Chapter worker finished!')