def __extractIntentionsFile(self, fn, data_dir, out_fn):
        self.logger.info('Making intention level from: %s', fn)
        if not os.path.exists(fn):
            self.logger.warn('File does not exists, skipping: %s', fn)
            return
        mlf = ConceptMLF.readFromFile(fn)
        fw = MLF()
        processor = NadraziProcessor([data_dir])
        for mlf_fn, tree in mlf.iteritems():
            fw[mlf_fn] = ['%s\n' % processor.process(mlf_fn, tree)]

        dirname = os.path.split(fn)[0]
        fw.writeToFile(os.path.join(dirname, out_fn))
    def __extractIntentionsFile(self, fn, data_dir, out_fn):
        self.logger.info('Making intention level from: %s', fn)
        if not os.path.exists(fn):
            self.logger.warn('File does not exists, skipping: %s', fn)
            return
        mlf = ConceptMLF.readFromFile(fn)
        fw = MLF()
        processor = NadraziProcessor([data_dir])
        for mlf_fn, tree in mlf.iteritems():
            fw[mlf_fn] = ['%s\n' % processor.process(mlf_fn, tree)]

        dirname = os.path.split(fn)[0]
        fw.writeToFile(os.path.join(dirname, out_fn))
    def mlfparametrize(self,
                       mlf,
                       inDir=None,
                       outDir=None,
                       source_type='normalized',
                       target_type='decoded',
                       use_empty=False):
        self.setupDirs(inDir, outDir)
        foo, new_sen_file = tempfile.mkstemp()
        self.logger.info("Reading MLF")
        mlf = MLF.readFromFile(mlf)
        self.logger.info("Making build directories")
        self.makeDirs()
        self.logger.info("Getting text from XMLs")
        if new_sen_file is None:
            fn = os.path.join(self.settings['PDT_WORK_DIR'], 'sen_file')
        else:
            fn = new_sen_file
        fw = codecs.open(fn, 'w', 'utf-8')
        try:
            for fn in self.getInputXMLs():
                dxml = DXML.readFromFile(fn)
                acts = dxml.getDialogueActs(source_type)
                idx = 0
                for utter in acts:
                    for txt, attrs in utter:
                        if not txt:
                            self.logger.info(
                                "Empty <dialogue_act> in file: %s", fn)
                        key = "*/%s_%05d" % (os.path.splitext(
                            os.path.basename(fn))[0], idx)
                        if key in mlf:
                            line = mlf[key]
                        elif not use_empty:
                            line = [txt + '\n']
                        else:
                            line = ['\n']
                        line = '\n'.join(line)
                        fw.write(line)
                        idx += 1
        finally:
            fw.close()

        self.parametrize(target_type, new_sen_file)
        os.remove(new_sen_file)
    def mlfparametrize(self, mlf, inDir=None, outDir=None,
            source_type='normalized', target_type='decoded', use_empty=False):
        self.setupDirs(inDir, outDir)
        foo, new_sen_file = tempfile.mkstemp()
        self.logger.info("Reading MLF")
        mlf = MLF.readFromFile(mlf)
        self.logger.info("Making build directories")
        self.makeDirs()
        self.logger.info("Getting text from XMLs")
        if new_sen_file is None:
            fn = os.path.join(self.settings['PDT_WORK_DIR'], 'sen_file')
        else:
            fn = new_sen_file
        fw = codecs.open(fn, 'w', 'utf-8')
        try:
            for fn in self.getInputXMLs():
                dxml = DXML.readFromFile(fn)
                acts = dxml.getDialogueActs(source_type)
                idx = 0
                for utter in acts:
                    for txt, attrs in utter:
                        if not txt:
                            self.logger.info("Empty <dialogue_act> in file: %s", fn)
                        key = "*/%s_%05d" % (os.path.splitext(os.path.basename(fn))[0], idx)
                        if key in mlf:
                            line = mlf[key]
                        elif not use_empty:
                            line = [txt + '\n']
                        else:
                            line = ['\n']
                        line = '\n'.join(line)
                        fw.write(line)
                        idx += 1
        finally:
            fw.close()

        self.parametrize(target_type, new_sen_file)
        os.remove(new_sen_file)
    def main(
        self,
        model_dir,
        encoding=None,
        batch=False,
        omit_leaves=False,
        mlf=False,
        xml_dir=None,
        ref_mlf=None,
        skip_empty=False,
        input_chain=None,
        batch_size=100,
        no_underscores=True,
        force_pdt=False,
        pdt_dir=None,
    ):
        encoding = sys.stdout.encoding
        if encoding is None:
            if os.name == "nt":
                encoding = "cp1250"
            else:
                encoding = "iso-8859-2"

        datasets_fn = pjoin(model_dir, "datasets")
        datasets_fr = file(datasets_fn, "r")
        datasets = []
        isymMaps = []
        for i, line in enumerate(datasets_fr):
            line = line.strip()
            datasets.append(line)
            if line != "off":
                isymMaps.append(SymMap.readFromFile(pjoin(model_dir, "isym%d.map" % (i + 1,))))

        osymMap = SymMap.readFromFile(pjoin(model_dir, "osym.map"))

        if "signed" in datasets:
            da_type = "signed"
        else:
            da_type = "normalized"

        if not pdt_dir:
            pdt_dir = "/opt/PDT-2.0/tools/machine-annotation"

        if xml_dir:
            reader = input.MultiReader([xml_dir], input.DXMLReader)
            if force_pdt and "lemma" in datasets or "pos" in datasets:
                if os.name == "nt":
                    raise RuntimeError("Datasets 'lemma' and 'pos' are unsupported on Windows")
                reader = input.PDTReader(pdt_dir, reader, online=not batch)
        else:
            reader = input.StdInReader(encoding=encoding, type=da_type)
            if "lemma" in datasets or "pos" in datasets:
                if os.name == "nt":
                    raise RuntimeError("Datasets 'lemma' and 'pos' are unsupported on Windows")
                reader = input.PDTReader(pdt_dir, reader, online=not batch)
        if input_chain is not None:
            reader = input.InputChain(input_chain, reader)
        generator = input.InputGenerator(reader, datasets, datasets[0], noUnderscores=no_underscores)
        hypMLF = MLF()
        refMLF = MLF()
        if not batch:
            for da_fn, da_id, da_semantics, da_txts in generator.readInputs():
                da_empty = not bool(da_semantics.strip())
                if da_empty and skip_empty:
                    continue

                refMLF[da_id] = da_semantics + "\n"
                dcd = self.parseLine(model_dir, [da_txts], isymMaps, osymMap, omitLeaves=omit_leaves)
                if dcd:
                    if len(dcd) == 1:
                        hypMLF[da_id] = dcd[0].encode(encoding) + "\n"
                    else:
                        hypMLF[da_id] = ";".join(dcd).encode(encoding) + "\n"
                else:
                    hypMLF[da_id] = line + "\n"
                if not mlf:
                    print hypMLF[da_id],
        else:
            all_processed = False
            inputs = generator.readInputs()
            while not all_processed:
                da_count = 0
                lines = []
                ids = []
                for da_fn, da_id, da_semantics, da_txts in inputs:
                    da_empty = not bool(da_semantics.strip())
                    if da_empty and skip_empty:
                        continue

                    refMLF[da_id] = da_semantics + "\n"
                    lines.append(da_txts)
                    ids.append(da_id)
                    da_count += 1
                    if da_count >= batch_size:
                        break
                else:
                    all_processed = True

                dcd = self.parseLine(model_dir, lines, isymMaps, osymMap, omitLeaves=omit_leaves)
                for da_id, ol in zip(ids, dcd):
                    hypMLF[da_id] = ol.encode(encoding) + "\n"
                    if not mlf:
                        print hypMLF[da_id],
        if mlf:
            s = "".join(hypMLF.toLines())
            print s

        if ref_mlf:
            refMLF.writeToFile(ref_mlf)
    def main(self,
             model_dir,
             encoding=None,
             batch=False,
             omit_leaves=False,
             mlf=False,
             xml_dir=None,
             ref_mlf=None,
             skip_empty=False,
             input_chain=None,
             batch_size=100,
             no_underscores=True,
             force_pdt=False,
             pdt_dir=None):
        encoding = sys.stdout.encoding
        if encoding is None:
            if os.name == 'nt':
                encoding = 'cp1250'
            else:
                encoding = 'iso-8859-2'

        datasets_fn = pjoin(model_dir, 'datasets')
        datasets_fr = file(datasets_fn, 'r')
        datasets = []
        isymMaps = []
        for i, line in enumerate(datasets_fr):
            line = line.strip()
            datasets.append(line)
            if line != 'off':
                isymMaps.append(
                    SymMap.readFromFile(
                        pjoin(model_dir, 'isym%d.map' % (i + 1, ))))

        osymMap = SymMap.readFromFile(pjoin(model_dir, 'osym.map'))

        if 'signed' in datasets:
            da_type = 'signed'
        else:
            da_type = 'normalized'

        if not pdt_dir:
            pdt_dir = '/opt/PDT-2.0/tools/machine-annotation'

        if xml_dir:
            reader = input.MultiReader([xml_dir], input.DXMLReader)
            if force_pdt and 'lemma' in datasets or 'pos' in datasets:
                if os.name == 'nt':
                    raise RuntimeError(
                        "Datasets 'lemma' and 'pos' are unsupported on Windows"
                    )
                reader = input.PDTReader(pdt_dir, reader, online=not batch)
        else:
            reader = input.StdInReader(encoding=encoding, type=da_type)
            if 'lemma' in datasets or 'pos' in datasets:
                if os.name == 'nt':
                    raise RuntimeError(
                        "Datasets 'lemma' and 'pos' are unsupported on Windows"
                    )
                reader = input.PDTReader(pdt_dir, reader, online=not batch)
        if input_chain is not None:
            reader = input.InputChain(input_chain, reader)
        generator = input.InputGenerator(reader,
                                         datasets,
                                         datasets[0],
                                         noUnderscores=no_underscores)
        hypMLF = MLF()
        refMLF = MLF()
        if not batch:
            for da_fn, da_id, da_semantics, da_txts in generator.readInputs():
                da_empty = not bool(da_semantics.strip())
                if (da_empty and skip_empty):
                    continue

                refMLF[da_id] = da_semantics + '\n'
                dcd = self.parseLine(model_dir, [da_txts],
                                     isymMaps,
                                     osymMap,
                                     omitLeaves=omit_leaves)
                if dcd:
                    if len(dcd) == 1:
                        hypMLF[da_id] = dcd[0].encode(encoding) + '\n'
                    else:
                        hypMLF[da_id] = ';'.join(dcd).encode(encoding) + '\n'
                else:
                    hypMLF[da_id] = line + '\n'
                if not mlf:
                    print hypMLF[da_id],
        else:
            all_processed = False
            inputs = generator.readInputs()
            while not all_processed:
                da_count = 0
                lines = []
                ids = []
                for da_fn, da_id, da_semantics, da_txts in inputs:
                    da_empty = not bool(da_semantics.strip())
                    if (da_empty and skip_empty):
                        continue

                    refMLF[da_id] = da_semantics + '\n'
                    lines.append(da_txts)
                    ids.append(da_id)
                    da_count += 1
                    if da_count >= batch_size:
                        break
                else:
                    all_processed = True

                dcd = self.parseLine(model_dir,
                                     lines,
                                     isymMaps,
                                     osymMap,
                                     omitLeaves=omit_leaves)
                for da_id, ol in zip(ids, dcd):
                    hypMLF[da_id] = ol.encode(encoding) + '\n'
                    if not mlf:
                        print hypMLF[da_id],
        if mlf:
            s = ''.join(hypMLF.toLines())
            print s

        if ref_mlf:
            refMLF.writeToFile(ref_mlf)