def convert(self):
""" reads the old cart tree and converts it to the new xml-format
"""
self._writer = openXml(self._pathToNewTree, self._newEncoding)
self._setNumberOfCartClasses()
self._reader = uopen(self._pathToOldTree)
self._writer.open("decision-tree")
self._convertPhonemeList()
self._convertQuestionList()
self._convertBinaryTree()
self._writer.close("decision-tree")
closeXml(self._writer)
uclose(self._reader)
def ensureMissingWordList(self): if not self.missingWordFile: raise 'cannot get missing words' if self.exists(self.missingWordFile): print >> sys.stderr, 'missing word file does already exist:', self.missingWordFile else: if self.setCostaLog(): print >> sys.stderr, 'get missing words from costa-log-file', self.costaLogFile self.missingWords = self.costaLog.getMissingWordList() fd = uopen(self.missingWordFile, self.encoding, 'w') print >> sys.stderr, 'store missing words:', fd.name for word in self.missingWords: fd.write(word + '\n') uclose(fd) else: raise 'cannot create missing word file', self.missingWordFile
def _setNumberOfCartClasses(self):
""" reads old cart file and determines the maximal index of a cart class
this is needed, because the index of the silence class is set to this value """
self._reader = uopen(self._pathToOldTree)
reachedTree = False
while True:
text = self._reader.readline().strip()
if not reachedTree:
if len(text) > 0:
if text[0] == '(':
reachedTree = True
elif text == '':
break
else:
text = text.strip("()")
tupel = map(int, text.split(','))
if len(tupel) == 2:
self._maxIndexOfCartClasses = max(
self._maxIndexOfCartClasses, tupel[0])
uclose(self._reader)
def closeXml(xml):
assert xml.elem.father == None
uclose(xml.fd)
def closeXml(xml):
xml.end()
uclose(xml.fd)
def main(argv):
defaultEncoding = "utf8"
usage = "usage: %prog [options] <corpus name> <corpus outfile>\n " + __doc__
optionParser = OptionParser(usage=usage)
optionParser.add_option("-E",
"--encoding",
default=defaultEncoding,
dest="encoding",
help="encoding [" + defaultEncoding + "]")
optionParser.add_option(
"-f",
"--file",
dest="sourcefilename",
help=
"source filename with at least sentence IDs and orthographies specified in a file"
)
optionParser.add_option("-F",
"--Field",
default="orth",
dest="orthfieldname",
help="orth field name, e.g. orth or zip [orth]")
optionParser.add_option("-d",
"--delimiter",
default=";",
dest="delimiter",
help="field delimiter in the plain file [;]")
optionParser.add_option(
"-a",
"--all",
dest="splitall",
action="store_true",
help="split every record into a single corpus file")
optionParser.add_option("-w",
"--wordcount",
dest="wordlist",
action="store_true",
help="print word list")
optionParser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true")
(options, args) = optionParser.parse_args()
if len(args) != 2:
optionParser.error("incorrect number of arguments %d" % len(args))
sys.exit()
#set filenames
corpusname = args[0]
corpusFilename = args[1]
if not corpusFilename.rfind(".corpus"):
corpusFilename += ".corpus"
recordingsFilename = corpusFilename[:corpusFilename.
rfind('.')] + ".recordings"
speakerDescriptionFilename = corpusFilename[:corpusFilename.
rfind('.')] + ".speaker"
if options.verbose:
print "corpusFilename =", corpusFilename
print "speakerDescriptionFilename =", speakerDescriptionFilename
print "recordingsFilename =", recordingsFilename
#create corpus include structure
newCorpus = Corpus(corpusname, encoding="utf8")
newCorpus.includeFile(os.path.abspath(speakerDescriptionFilename))
newCorpus.includeFile(os.path.abspath(recordingsFilename))
newCorpus.save(corpusFilename)
# create default speaker description file
speakers = []
speakerDescription = Speaker(corpusname, encoding="utf8")
#read sentence IDs and structure information
sentenceIDsFile = uopen(options.sourcefilename, options.encoding, 'r')
# sentenceIDsFile = open(options.sourcefilename,'r')
firstLine = sentenceIDsFile.readline()[:-1]
# fieldList = unicode(firstLine, options.encoding).split(options.delimiter)
fieldList = firstLine.split(options.delimiter)
if options.verbose:
print "structure:", firstLine, fieldList
fieldMap = {}
fieldId = 0
for field in fieldList:
fieldMap[field] = fieldId
if options.verbose:
print field, fieldId
fieldId += 1
#check required fields
if not fieldMap.has_key('name') or not fieldMap.has_key(
'video') or not fieldMap.has_key('orth'):
print "ERROR: one or more required fields [name,video, and/or orth] are missing."
keys = fieldMap.keys()
keys.sort
for key in keys:
print key, fieldMap[key]
sys.exit()
#create recordings from data info file
newRecordings = Corpus(corpusname, encoding="utf-8")
sentenceCnt = 0
for line in sentenceIDsFile:
sentenceCnt += 1
# splitlist = unicode(line, options.encoding).strip().split(options.delimiter)
splitlist = line.strip().split(options.delimiter)
if options.verbose:
for i in range(0, len(splitlist)):
print i, splitlist[i]
if len(splitlist) < len(fieldMap):
if options.verbose:
print "ERROR: data row '%s' is invalid and will be discarded." % (
splitlist)
else:
start = -1
end = -1
speakerName = "default"
speakerGender = "male"
recordOrth = ""
recordTranslation = ""
if (fieldMap.get("start") != None):
start = splitlist[fieldMap['start']]
if (fieldMap.get("end") != None):
end = splitlist[fieldMap['end']]
if (fieldMap.get("speaker") != None):
speakerName = splitlist[fieldMap['speaker']]
if (fieldMap.get("gender") != None):
speakerGender = splitlist[fieldMap['gender']]
if (fieldMap.get("translation") != None):
recordTranslation = splitlist[fieldMap['translation']]
# update speaker names
if speakerName not in speakers:
speakerDescription.addSpeakerDescription(
speakerName, speakerGender)
speakers.append(speakerName)
# add recording
newRecordings.addRecording(
splitlist[fieldMap['name']], splitlist[fieldMap['video']],
start, end, speakerName,
splitlist[fieldMap[options.orthfieldname]], False,
recordTranslation)
# close corpus and write to xml file
uclose(sentenceIDsFile)
newRecordings.save(recordingsFilename)
speakerDescription.save(speakerDescriptionFilename)
print "\n----------------------------------------------------------"
print "corpus file :'" + corpusFilename + "'"
print "speaker description file :'" + speakerDescriptionFilename + "'"
print "recordings file :'" + recordingsFilename + "'"
print
def save(self, filename):
out = uopen(filename, self.encoding, "w")
self.doc.writexml(out, '', ' ', '\n', 'utf-8')
uclose(out)
def save(self, filename):
out = uopen(filename, self.encoding, 'w')
self.doc.writexml(out, '', ' ', '\n', self.encoding)
uclose(out)
def suspendXml(xml):
assert xml.elem.father is not None and xml.elem.father.father is None
uclose(xml.fd)