args = parser.parse_args()
# ----------------------------------------------------------------------------
if not args.quiet:
setup_logging(1,None)
# ----------------------------------------------------------------------------
# Automatic Tokenization is here:
# ----------------------------------------------------------------------------
base = os.path.basename( args.vocab )
lang = base[:3]
if args.i:
p = sppasTok( args.vocab,lang )
if args.std:
p.set_std(True)
p.run( args.i, args.o )
else:
vocab = WordsList( args.vocab )
tokenizer = DictTok( vocab,lang )
try:
repl = DictRepl(os.path.join(RESOURCES_PATH, "repl", lang + ".repl"), nodump=True)
tokenizer.set_repl( repl )
except Exception as e:
print "[warning] No replacement dictionary: ",str(e)
try:
def run_tokenization(self, stepidx):
"""
Execute the SPPAS-Tokenization program.
@return number of files processed successfully
"""
# Initializations
step = self.parameters.get_step(stepidx)
stepname = self.parameters.get_step_name(stepidx)
files_processed_success = 0
self._progress.set_header(stepname)
self._progress.update(0,"")
# Get the list of input file names, with the ".wav" (or ".wave") extension
filelist = self.set_filelist(".wav")#,not_start=["track_"])
if len(filelist) == 0:
return 0
total = len(filelist)
# Create annotation instance
try:
self._progress.set_text("Loading resources...")
t = sppasTok( step.get_langresource(), logfile=self._logfile, lang=step.get_lang() )
except Exception as e:
if self._logfile is not None:
self._logfile.print_message( "%s\n"%str(e), indent=1,status=4 )
return 0
# Execute the annotation for each file in the list
for i,f in enumerate(filelist):
# fix the default values
t.fix_options( step.get_options() )
# Indicate the file to be processed
self._progress.set_text( os.path.basename(f)+" ("+str(i+1)+"/"+str(total)+")" )
if self._logfile is not None:
self._logfile.print_message(stepname+" of file " + f, indent=1 )
# Get the input file
inname = self._get_filename(f, [self.parameters.get_output_format()] + annotationdata.io.extensions_out)
if inname is not None:
# Fix output file name
outname = os.path.splitext(f)[0] + '-token' + self.parameters.get_output_format()
# Execute annotation
try:
t.run( inname, outname )
except Exception as e:
if self._logfile is not None:
self._logfile.print_message( "%s for file %s\n"%(str(e),outname), indent=2,status=-1 )
else:
files_processed_success += 1
if self._logfile is not None:
self._logfile.print_message(outname, indent=2,status=0 )
else:
if self._logfile is not None:
self._logfile.print_message("Failed to find a file with transcription. Read the documentation for details.",indent=2,status=2)
# Indicate progress
self._progress.set_fraction(float((i+1))/float(total))
if self._logfile is not None:
self._logfile.print_newline()
# Indicate completed!
self._progress.update(1,"Completed (%d files successfully over %d files).\n"%(files_processed_success,total))
self._progress.set_header("")
return files_processed_success