Example #1
0
    def _ingest_token(self, token_entry):
        sidx = Corpus._ingest_token(self, token_entry)
        if (math.isnan(sidx)):
            return None

        lang_pair = token_entry['lang_pair']
        self._add_lang_pair(lang_pair)

        cslabel = token_entry['cs_label']
        self._add_cslabel(cslabel, sidx)

        cstype = token_entry['cs_type']
        self._add_cstype(cstype, sidx)