def execute(self): logging.debug("Verse worker started!") self.clear_report() self.__temp_dir = init_temp_dir() for src_file in self.__ftp_dir.rglob('*.wav'): # Process verse/chunk files only if not re.search(self.__verse_regex, str(src_file)): continue # Extract necessary path parts root_parts = self.__ftp_dir.parts parts = src_file.parts[len(root_parts):] lang = parts[0] resource = parts[1] book = parts[2] chapter = parts[3] media = parts[5] grouping = parts[7] if media == 'mp3' else parts[6] target_dir = self.__temp_dir.joinpath(lang, resource, book, chapter, grouping) remote_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS") target_dir.mkdir(parents=True, exist_ok=True) target_file = target_dir.joinpath(src_file.name) logging.debug(f'Found verse file: {src_file}') mp3_exists = check_file_exists(src_file, remote_dir, 'mp3', grouping) cue_exists = check_file_exists(src_file, remote_dir, 'cue', grouping) if mp3_exists and cue_exists: logging.debug(f'Files exist. Skipping...') continue # Copy source file to temp dir logging.debug(f'Copying file {src_file} to {target_file}') target_file.write_bytes(src_file.read_bytes()) # Try to fix wav metadata logging.debug(f'Fixing metadata: {target_file}') fix_metadata(target_file, self.verbose) # Convert verse into mp3 self.convert_verse(target_file, remote_dir, grouping) logging.debug(f'Deleting temporary directory {self.__temp_dir}') rm_tree(self.__temp_dir) logging.debug('Verse worker finished!')
def delete_verse_files(self, lang, resource, book, chapter): """ Delete verse related files (wav, mp3, cue) """ remote_chapter_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS") wav_verse_files = remote_chapter_dir.joinpath("wav", "verse") if wav_verse_files.exists(): rm_tree(wav_verse_files) self.resources_deleted.append(str(rel_path(wav_verse_files, self.__ftp_dir))) mp3_hi_verse_dir = remote_chapter_dir.joinpath("mp3", "hi", "verse") if mp3_hi_verse_dir.exists(): rm_tree(mp3_hi_verse_dir) self.resources_deleted.append(str(rel_path(mp3_hi_verse_dir, self.__ftp_dir))) mp3_low_verse_dir = remote_chapter_dir.joinpath("mp3", "low", "verse") if mp3_low_verse_dir.exists(): rm_tree(mp3_low_verse_dir) self.resources_deleted.append(str(rel_path(mp3_low_verse_dir, self.__ftp_dir))) cue_verse_dir = remote_chapter_dir.joinpath("cue", "verse") if cue_verse_dir.exists(): rm_tree(cue_verse_dir) self.resources_deleted.append(str(rel_path(cue_verse_dir, self.__ftp_dir)))
def delete_chapter_files(self, lang, resource, book, chapter): """ Delete chapter related files (mp3 and cue) """ remote_chapter_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS") mp3_hi_chapter_dir = remote_chapter_dir.joinpath("mp3", "hi", "chapter") if mp3_hi_chapter_dir.exists(): rm_tree(mp3_hi_chapter_dir) self.resources_deleted.append(str(rel_path(mp3_hi_chapter_dir, self.__ftp_dir))) mp3_low_chapter_dir = remote_chapter_dir.joinpath("mp3", "low", "chapter") if mp3_low_chapter_dir.exists(): rm_tree(mp3_low_chapter_dir) self.resources_deleted.append(str(rel_path(mp3_low_chapter_dir, self.__ftp_dir))) cue_chapter_dir = remote_chapter_dir.joinpath("cue", "chapter") if cue_chapter_dir.exists(): rm_tree(cue_chapter_dir) self.resources_deleted.append(str(rel_path(cue_chapter_dir, self.__ftp_dir)))
def execute(self): """ Execute worker """ logging.debug("TR worker started!") self.clear_report() self.__temp_dir = init_temp_dir() existent_tr = self.find_existent_tr() media = ['wav', 'mp3/hi', 'mp3/low'] for m in media: for src_file in self.__ftp_dir.rglob(f'{m}/verse/*.*'): if src_file.suffix == '.tr': continue # Process verse files only if not re.search(self.__verse_regex, str(src_file)): continue self.__book_tr_files.append(src_file) self.__chapter_tr_files.append(src_file) # Extract necessary path parts root_parts = self.__ftp_dir.parts parts = src_file.parts[len(root_parts):] lang = parts[0] resource = parts[1] book = parts[2] chapter = parts[3] media = parts[5] quality = parts[6] if media == 'mp3' else '' grouping = parts[7] if media == 'mp3' else parts[6] regex = fr'{lang}\/{resource}\/{book}(?:\/{chapter})?\/' \ fr'CONTENTS\/tr\/{media}(?:\/{quality})?\/{grouping}' for group, tr in existent_tr: if not re.search(regex, str(tr)): continue if group == Group.BOOK and src_file in self.__book_tr_files: self.__book_tr_files.remove(src_file) elif group == Group.CHAPTER and src_file in self.__chapter_tr_files: self.__chapter_tr_files.remove(src_file) # Create chapter TRs chapter_groups = self.group_files(self.__chapter_tr_files, Group.CHAPTER) for key in chapter_groups: self.create_tr_file(key, chapter_groups[key]) # Create book TRs book_groups = self.group_files(self.__book_tr_files, Group.BOOK) for key in book_groups: self.create_tr_file(key, book_groups[key]) logging.debug(f'Deleting temporary directory {self.__temp_dir}') rm_tree(self.__temp_dir) logging.debug('TR worker finished!')
def create_tr_file(self, dic: str, files: List[Path]): """ Create tr file and copy it to the remote directory""" parts = json.loads(dic) lang = parts['lang'] resource = parts['resource'] book = parts['book'] chapter = parts['chapter'] if 'chapter' in parts else None media = parts['media'] quality = parts['quality'] grouping = parts['grouping'] root_dir = self.__temp_dir.joinpath('root') target_dir = root_dir.joinpath(lang, resource, book) if chapter is not None: remote_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS") else: remote_dir = self.__ftp_dir.joinpath(lang, resource, book, "CONTENTS") for file in files: target_chapter = chapter if target_chapter is None: match = re.search(r'_c([0-9]+)_v[0-9]+', file.name) if not match: raise Exception( 'Could not define chapter from the file name.') target_chapter = match.group(1) target_chapter_dir = target_dir.joinpath( self.zero_pad_chapter(target_chapter, book)) target_chapter_dir.mkdir(parents=True, exist_ok=True) target_file = target_chapter_dir.joinpath(file.name) # Copy source file to temp dir logging.debug(f'Copying file {file} to {target_file}') target_file.write_bytes(file.read_bytes()) # Create TR file logging.debug('Creating TR file') create_tr(root_dir, self.verbose) tr = self.__temp_dir.joinpath('root.tr') if chapter is not None: new_tr = Path(tr.parent, f'{lang}_{resource}_{book}_c{chapter}.tr') else: new_tr = Path(tr.parent, f'{lang}_{resource}_{book}.tr') tr.rename(new_tr) # Copy tr file to remote dir logging.debug(f'Copying {new_tr} to {remote_dir}') t_file = copy_file(new_tr, remote_dir, grouping, quality, media) self.resources_created.append(str(rel_path(t_file, self.__ftp_dir))) rm_tree(root_dir) new_tr.unlink()
def execute(self): """ Execute worker """ logging.debug("Chapter worker started!") self.clear_report() self.__temp_dir = init_temp_dir() for src_file in self.__ftp_dir.rglob('*.wav'): # Process chapter files only if not re.search(self.__chapter_regex, str(src_file)): continue # Extract necessary path parts root_parts = self.__ftp_dir.parts parts = src_file.parts[len(root_parts):] lang = parts[0] resource = parts[1] book = parts[2] chapter = parts[3] grouping = parts[6] target_dir = self.__temp_dir.joinpath(lang, resource, book, chapter, grouping) remote_dir = self.__ftp_dir.joinpath(lang, resource, book, chapter, "CONTENTS") verses_dir = target_dir.joinpath("verses") verses_dir.mkdir(parents=True, exist_ok=True) target_file = target_dir.joinpath(src_file.name) logging.debug(f'Found chapter file: {src_file}') # Copy source file to temp dir logging.debug(f'Copying file {src_file} to {target_file}') target_file.write_bytes(src_file.read_bytes()) # Try to fix wav metadata logging.debug(f'Fixing metadata: {target_file}') fix_metadata(target_file, self.verbose) # Split chapter files into verses logging.debug(f'Splitting chapter {target_file} into {verses_dir}') split_chapter(target_file, verses_dir, self.verbose) target_verse_dir = remote_dir.joinpath("wav", "verse") is_new = check_dir_empty(target_verse_dir) is_changed = has_new_files(verses_dir, target_verse_dir) cleaning = is_new or is_changed # If we have a new or updated chapter WAV file # delete all the chapter related resources: # split verses, converted files and TR files if cleaning: # Delete related chapter files self.delete_chapter_files(lang, resource, book, chapter) # Delete related verse files self.delete_verse_files(lang, resource, book, chapter) # Delete related book TR files self.delete_tr_book_files(lang, resource, book) # Delete related chapter TR files self.delete_tr_chapter_files(lang, resource, book, chapter) # Copy original verse files logging.debug( f'Copying original verse files from {verses_dir} into {remote_dir}' ) t_dir = copy_dir(verses_dir, remote_dir) if cleaning: self.resources_created.append(str(rel_path(t_dir, self.__ftp_dir))) # Convert chapter into mp3 self.convert_chapter(target_file, remote_dir, grouping) # Convert verses into mp3 self.convert_verses(verses_dir, remote_dir) logging.debug(f'Deleting temporary directory {self.__temp_dir}') rm_tree(self.__temp_dir) logging.debug('Chapter worker finished!')