def main(argv): defaultEncoding = "utf8" usage = "usage: %prog [options] <corpus name> <corpus outfile>\n " + __doc__ optionParser = OptionParser(usage=usage) optionParser.add_option("-E", "--encoding", default=defaultEncoding, dest="encoding", help="encoding [" + defaultEncoding + "]") optionParser.add_option( "-f", "--file", dest="sourcefilename", help= "source filename with at least sentence IDs and orthographies specified in a file" ) optionParser.add_option("-F", "--Field", default="orth", dest="orthfieldname", help="orth field name, e.g. orth or zip [orth]") optionParser.add_option("-d", "--delimiter", default=";", dest="delimiter", help="field delimiter in the plain file [;]") optionParser.add_option( "-a", "--all", dest="splitall", action="store_true", help="split every record into a single corpus file") optionParser.add_option("-w", "--wordcount", dest="wordlist", action="store_true", help="print word list") optionParser.add_option("-v", "--verbose", dest="verbose", action="store_true") (options, args) = optionParser.parse_args() if len(args) != 2: optionParser.error("incorrect number of arguments %d" % len(args)) sys.exit() #set filenames corpusname = args[0] corpusFilename = args[1] if not corpusFilename.rfind(".corpus"): corpusFilename += ".corpus" recordingsFilename = corpusFilename[:corpusFilename. rfind('.')] + ".recordings" speakerDescriptionFilename = corpusFilename[:corpusFilename. rfind('.')] + ".speaker" if options.verbose: print "corpusFilename =", corpusFilename print "speakerDescriptionFilename =", speakerDescriptionFilename print "recordingsFilename =", recordingsFilename #create corpus include structure newCorpus = Corpus(corpusname, encoding="utf8") newCorpus.includeFile(os.path.abspath(speakerDescriptionFilename)) newCorpus.includeFile(os.path.abspath(recordingsFilename)) newCorpus.save(corpusFilename) # create default speaker description file speakers = [] speakerDescription = Speaker(corpusname, encoding="utf8") #read sentence IDs and structure information sentenceIDsFile = uopen(options.sourcefilename, options.encoding, 'r') # sentenceIDsFile = open(options.sourcefilename,'r') firstLine = sentenceIDsFile.readline()[:-1] # fieldList = unicode(firstLine, options.encoding).split(options.delimiter) fieldList = firstLine.split(options.delimiter) if options.verbose: print "structure:", firstLine, fieldList fieldMap = {} fieldId = 0 for field in fieldList: fieldMap[field] = fieldId if options.verbose: print field, fieldId fieldId += 1 #check required fields if not fieldMap.has_key('name') or not fieldMap.has_key( 'video') or not fieldMap.has_key('orth'): print "ERROR: one or more required fields [name,video, and/or orth] are missing." keys = fieldMap.keys() keys.sort for key in keys: print key, fieldMap[key] sys.exit() #create recordings from data info file newRecordings = Corpus(corpusname, encoding="utf-8") sentenceCnt = 0 for line in sentenceIDsFile: sentenceCnt += 1 # splitlist = unicode(line, options.encoding).strip().split(options.delimiter) splitlist = line.strip().split(options.delimiter) if options.verbose: for i in range(0, len(splitlist)): print i, splitlist[i] if len(splitlist) < len(fieldMap): if options.verbose: print "ERROR: data row '%s' is invalid and will be discarded." % ( splitlist) else: start = -1 end = -1 speakerName = "default" speakerGender = "male" recordOrth = "" recordTranslation = "" if (fieldMap.get("start") != None): start = splitlist[fieldMap['start']] if (fieldMap.get("end") != None): end = splitlist[fieldMap['end']] if (fieldMap.get("speaker") != None): speakerName = splitlist[fieldMap['speaker']] if (fieldMap.get("gender") != None): speakerGender = splitlist[fieldMap['gender']] if (fieldMap.get("translation") != None): recordTranslation = splitlist[fieldMap['translation']] # update speaker names if speakerName not in speakers: speakerDescription.addSpeakerDescription( speakerName, speakerGender) speakers.append(speakerName) # add recording newRecordings.addRecording( splitlist[fieldMap['name']], splitlist[fieldMap['video']], start, end, speakerName, splitlist[fieldMap[options.orthfieldname]], False, recordTranslation) # close corpus and write to xml file uclose(sentenceIDsFile) newRecordings.save(recordingsFilename) speakerDescription.save(speakerDescriptionFilename) print "\n----------------------------------------------------------" print "corpus file :'" + corpusFilename + "'" print "speaker description file :'" + speakerDescriptionFilename + "'" print "recordings file :'" + recordingsFilename + "'" print