def _cache_IETF(cls): cls.precache(include=['script_expr_txt', 'region_expr_txt', 'region_expr_uid'], all_lv=True) for uid in cls._cache: cls._cache[uid]['IETF'] = set() result = panlex.query_all('/expr', {'trans_uid': 'art-420', 'uid': 'art-274', 'include': 'trans_txt'})['result'] regions = {cls._cache[r['txt']]['region_expr'] for r in result} regions_result = panlex.query_all('/expr', {'trans_expr': list(regions), 'uid': 'art-006', 'include': 'trans_txt'})['result'] regions_dict = {r['trans_txt']: r['txt'] for r in regions_result if len(r['txt']) == 2} for r in result: uid = r['txt'] lang = cls._cache[uid] given_tag = Language.get(r['trans_txt'], normalize=False) normalized_tag = Language.get(r['trans_txt'], normalize=True) language_set = {lang['lang_code'], given_tag.language, normalized_tag.language} script_set = {lang['script_expr_txt'], given_tag.script, normalized_tag.script} region_set = {given_tag.region, normalized_tag.region} if lang['region_expr_uid'] == 'art-279' and lang['region_expr_txt'] != '001': region_set.add(lang['region_expr_txt']) try: region_set.add(regions_dict[lang['region_expr_txt']]) except KeyError: pass if {'GB', 'UK'} & region_set: region_set |= {'GB', 'UK'} if {'001', None} & region_set: region_set |= {'001', None} for language, script, region in product(language_set, script_set, region_set): new_tag = normalized_tag.update_dict({'language': language, 'script': script, 'region': region}) cls._cache[uid]['IETF'].add(str(new_tag)) if lang['region_expr_uid'] == 'art-279' and lang['region_expr_txt'] == '001': for language, script, region in product({lang['lang_code']}, script_set, {'001', None}): new_tag = normalized_tag.update_dict({'language': language, 'script': script, 'region': region}) cls._cache[uid]['IETF'].add(str(new_tag))
def _set_lang_codes(self): with open (os.path.join(os.path.dirname(__file__), 'locale.json')) as f: data = json.load(f) lang = Language.get(self.lang).language terr = Language.get(self.lang).territory if terr is None: self.lang_codes = self._get_lang_codes(lang, data) else: self.lang_codes = [self.lang.replace('-', '_')]
def _create_user(self, user_id=None): lang_code = random.choice(['zh-hans', 'jp', 'ko', 'en', 'es']) language = Language.get(lang_code) user_id = utils.random_string()[0:10] if user_id is None else user_id param = { 'user_id': user_id, 'domain_id': self.domain.domain_id, 'password': '******', 'name': 'Steven' + utils.random_string()[0:5], 'language': language.__str__(), 'timezone': 'Asia/Seoul', 'tags': { 'aa': 'bb' }, 'email': 'Steven' + utils.random_string()[0:5] + '@mz.co.kr', 'mobile': '+821026671234', 'group': 'group-id', } user = self.identity_v1.User.create(param, metadata=(('token', self.token), )) self.user = user self.users.append(user) self.assertEqual(self.user.name, param['name'])
def test_create_user(self, user_id=None, name=None, user_type=None, backend=None): lang_code = random.choice(['zh-hans', 'jp', 'ko', 'en', 'es']) language = Language.get(lang_code) if user_id is None: user_id = utils.random_string() + '@mz.co.kr' if name is None: name = 'Steven' + utils.random_string() params = { 'user_id': user_id, 'password': utils.generate_password(), 'name': name, 'email': user_id, 'timezone': 'Asia/Seoul', 'language': language.__str__(), 'tags': { 'tag_key': 'tag_value' }, 'domain_id': self.domain.domain_id } user = self.identity_v1.User.create(params, metadata=(('token', self.owner_token), )) self.user = user self.users.append(user) self._print_data(self.user, 'test_create_user') self.assertEqual(self.user.name, params['name'])
def format_language(value: Optional[str]): lang = Language.get(value) if not lang.is_valid(): raise ValueError('language_invalid') return lang.simplify_script().to_tag()
async def prepareDataSet(self, data_set: NLUDataSet) -> None: last_exception = None # Try all language tag derivations, from specific to broad for language in Language.get( data_set.language).simplify_script().broaden(): language = language.to_tag() try: if not self.__skip_language_installations: self._logger.info( "Installing language resources for \"%s\"...", language) subprocess.run([ self.__python, "-m", "snips_nlu", "download", language ], check=True) self.__language = language last_exception = None break except BaseException as e: # pylint: disable=broad-except last_exception = e if last_exception is not None: raise last_exception
async def prepareDataSet(self, data_set: NLUDataSet) -> None: self.__language = Language.get(data_set.language).simplify_script().to_tag() agent_parent_path = self.__agents_client.project_path(self.__project) # The default language code doesn't really matter as this code always explicitly passes the # exact language on each step. Still, the default language code HAS to be set and it MUST # be set to the code that already is the default. # The following code attempts to retrieve the current agent and to extract the current # default language code from it. try: default_language_code = self.__agents_client.get_agent( agent_parent_path ).default_language_code except: # pylint: disable=bare-except # TODO: Unable to figure out which exact error is raised in case the agent doesn't # exist, which is why this code catches any exception that might be raised by the call # to get_agent. default_language_code = "en" self.__agents_client.set_agent(dialogflow_v2.types.Agent( parent = agent_parent_path, display_name = self.__agent, time_zone = self.__time_zone, default_language_code = default_language_code, supported_language_codes = [ self.__language ] ))
def test_create_user(self, user_id=None, name=None): lang_code = random.choice(['zh-hans', 'jp', 'ko', 'en', 'es']) language = Language.get(lang_code) if user_id is None: user_id = utils.random_string()[0:10] if name is None: name = 'Steven' + utils.random_string()[0:5] params = { 'user_id': user_id, 'password': '******', 'name': name, 'language': language.__str__(), 'tags': { 'key': 'value' }, 'email': 'Steven' + utils.random_string()[0:5] + '@mz.co.kr', 'mobile': '+821026671234', 'group': 'group-id', 'domain_id': self.domain.domain_id } user = self.identity_v1.User.create(params, metadata=(('token', self.token), )) self.user = user self.users.append(user) self._print_data(self.user, 'test_create_user') self.assertEqual(self.user.name, params['name'])
def from_IETF(tag, normalize=False): cache = cache_IETF() tag = str(Language.get(tag, normalize)) output = [] for uid in cache: if tag in cache[uid]['IETF']: output.append(uid) return output
async def _prepareDataSet(self, data_set: NLUDataSet) -> None: language = Language.get(data_set.language).language if self.__pipeline == "supervised": pipeline_config = "supervised_embeddings" image = "rasa/rasa:{}".format(self.__VERSION) if self.__pipeline == "pretrained": pipeline_config = "pretrained_embeddings_spacy" # In theory it should be enough to install rasa/rasa:latest-spacy-{language}, but in # practice the training fails in these images due to the spaCy models not being found. # This bug is reported in the Rasa repo: https://github.com/RasaHQ/rasa/issues/4789 image = "rasa/rasa:{}-spacy-{}".format(self.__VERSION, language) # Create the Rasa config self.__rasa_config_yml = yaml.dump({ "language": language, "pipeline": pipeline_config }) # Connect to the Docker daemon and pull the Rasa container self._logger.info("Preparing the docker container for Rasa...") self._logger.debug("Pulling Rasa image \"%s\"...", image) self.__docker.images.pull(image) self._logger.debug("Starting the Rasa HTTP server...") self.__container = self.__docker.containers.run( image, # Run the Rasa server and enable the HTTP API [ "run", "--enable-api" ], # Automatically remove the container after the server shuts down auto_remove=True, # Don't wait for the command to finish detach=True, # Expose port 5005 (used for HTTP by Rasa) for TCP traffic to a random port ports={ "5005/tcp": None } ) # Update the container information from the Docker daemon self.__container.reload() # Extract the port mapping and build the base url for the HTTP API port_mapping = self.__container.attrs["NetworkSettings"]["Ports"]["5005/tcp"][0] self.__url = "http://{}:{}/".format(port_mapping["HostIp"], port_mapping["HostPort"]) self._logger.debug("Waiting for the health endpoint to come alive...") for _ in range(self.__timeout): try: success = requests.get(self.__url).status_code == 200 except requests.exceptions.ConnectionError: success = False if success: break await asyncio.sleep(1) self._logger.info("Container running.")
async def _prepareDataSet(self, data_set: NLUDataSet) -> None: self.__app_id = self.__authoring_client.apps.add({ "name": "NLUTestFramework", "culture": Language.get(data_set.language).simplify_script().to_tag(), "initial_version_id": self.__class__.FAKE_VERSION })
def get_languages() -> List[Language]: langs = getattr(settings, "GARNETT_TRANSLATABLE_LANGUAGES", [get_default_language()]) if callable(langs): langs = langs() if type(langs) == list: return [Language.get(lang) for lang in langs] raise ImproperlyConfigured( "GARNETT_TRANSLATABLE_LANGUAGES must be a list or a callable that returns a list" )
def get_video_print(self, videos: List[Track]) -> List[List[str]]: if not videos: return [["--"]] data = [] for video in videos: codec = { "MPEG Video": f"MPEG-{(video.format_version or '').replace('Version ', '')}" }.get(video.format, video.format) scan_overview = video.scan_type vst = False if codec in ["MPEG-1", "MPEG-2"]: # parse d2v file with pyd2v, generates D2V if needed d2v = D2V.load(Path(self.file)) self.file = d2v.path # get every frames' flag data, this contains information on displaying frames # add vob and cell number to each frames flag data as well flags = [ f for line in [[ dict(**y, vob=x["vob"], cell=x["cell"]) for y in x["flags"] ] for x in d2v.data] for f in line ] interlaced_percent = ( sum(1 for f in flags if not f["progressive_frame"]) / len(flags)) * 100 if interlaced_percent == 100: scan_overview = "Interlaced (CST)" else: scan_overview = f"{round(interlaced_percent, 2)}% Interlaced (VST)" vst = True for ext in ["log", "d2v", "mpg", "mpeg"]: fp = os.path.splitext(self.file)[0] + "." + ext if os.path.exists(fp): os.unlink(fp) line_1 = "- {language}, {codec} ({profile}) {width}x{height} ({aspect}) @ {bitrate}".format( language=Language.get(video.language).display_name(), codec=codec, profile=video.format_profile, width=video.width, height=video.height, aspect=video.other_display_aspect_ratio[0], bitrate= f"{video.other_bit_rate[0]}{f' ({video.bit_rate_mode})' if video.bit_rate_mode else ''}" ) line_2 = " {fps} FPS ({fps_mode}), {color_space}{subsampling}P{bit_depth}, {scan}".format( fps=f"{video.framerate_num}/{video.framerate_den}" if video.framerate_num else video.frame_rate, fps_mode="VFR" if vst else video.frame_rate_mode, color_space=video.color_space, subsampling=video.chroma_subsampling.replace(":", ""), bit_depth=video.bit_depth, scan=scan_overview) data.append([line_1, line_2]) return data
def from_IETF(cls, tag, normalize=True): if cls._cache: try: [cls._cache[uid]['IETF'] for uid in cls._cache] except KeyError: cls._cache_IETF() else: cls._cache_IETF() tag = str(Language.get(tag, normalize)) output = [] for uid in cls._cache: if tag in cls._cache[uid]['IETF']: output.append(uid) return output
def get_language_from_request(request) -> Language: opt_order = getattr( settings, "GARNETT_REQUEST_LANGUAGE_SELECTORS", [ "garnett.selectors.query", "garnett.selectors.cookie", "garnett.selectors.header", ], ) for opt in opt_order: func = import_string(opt) if lang := func(request): return Language.get(lang)
def _setLanguage(self, language: str) -> None: """ Args: language: The language of this data set. Use this method to set the language, if the language was dynamically loaded from the data set itself. The language is represented by its ISO 639-1 code (e.g. "en"). Raises: :exc:`ValueError`: if the language was already set. """ if self.__language is not None: raise ValueError("The language for this data set was already set.") self.__language = Language.get(language).maximize().to_tag()
def get_default_language(): setting = getattr(settings, "GARNETT_DEFAULT_TRANSLATABLE_LANGUAGE", "en-AU") if callable(setting): default = setting() else: default = setting if isinstance(default, Language): return default elif isinstance(default, str): return Language.get(default) else: raise ImproperlyConfigured( "GARNETT_DEFAULT_TRANSLATABLE_LANGUAGE must be a string or callable that returns a string or `Language` object" )
def _create_user(self): self.user_param = { 'user_id': (utils.random_string()[0:10]), 'password': '******', 'name': 'Steven' + utils.random_string()[0:5], 'language': Language.get('jp').__str__(), 'timezone': 'utc+9', 'tags': { 'aa': 'bb' }, 'domain_id': self.domain.domain_id, 'email': 'Steven' + utils.random_string()[0:5] + '@mz.co.kr', 'mobile': '+821026671234', 'group': 'group-id', } self.user = self.identity_v1.User.create( self.user_param, metadata=(('token', self.owner_token), ))
def get_subtitle_print(subs: List[Track]) -> List[str]: """ Return a list of a brief subtitle overview per-subtitle. e.g. - English, Forced, SubRip (SRT) - English, SubRip (SRT) - English, SDH, SubRip (SRT) - Spanish, Latin American (SDH), SubRip (SRT) The bit of text between the Language and the Subtitle format is the Track Title. It can be of any format, but it is recommended to be used as shown above. It will be returned as a list of strings with the `- ` already pre-pended to each entry. """ data = [] if not subs: data.append("--") for sub in subs: line_items = [] # following sub.title tree checks and supports three different language and title scenarios # The second scenario is the recommended option to choose if you are open to choosing any # The third scenario should be used if you have nothing unique to state about the track # | Language | Track Title | Output | # | ------------ | ----------------------------- | --------------------------------------------- | # | es / Spanish | Spanish (Latin American, SDH) | - Spanish (Latin American, SDH), SubRip (SRT) | # | es / Spanish | Latin American (SDH) | - Spanish, Latin American (SDH), SubRip (SRT) | # | es / Spanish | None | - Spanish, SubRip (SRT) | language = Language.get(sub.language).display_name() if sub.title: if language.lower() in sub.title.lower(): line_items.append(sub.title) else: line_items.append(f"{language}, {sub.title}") else: line_items.append(language) line_items.append(sub.format.replace("UTF-8", "SubRip (SRT)")) line = "- " + ", ".join(line_items) data += [(" " + x if i > 0 else x) for i, x in enumerate(textwrap.wrap(line, 64))] return data
def runSimpleJSONDataSetTests(path, title, constructor_language, expected_language, size): expected_language = Language.get(expected_language).maximize().to_tag() # Run the tests twice, ignoring existing caches on the first run for ignore_cache in [ True, False ]: # Construct the data set data_set = SimpleJSONDataSet(title, path, 50, constructor_language, ignore_cache) assert data_set.title == title assert data_set.language == expected_language # Verify that the training data does not contain any None-intent sentences assert len(list(filter(lambda x: x.intent is None, data_set.training_data))) == 0 # Get the number of None-intent sentences in the validation data num_none_intent_sentences = len(list(filter( lambda x: x.intent is None, data_set.validation_data ))) # Verify that the training and validation data (without None-intent sentences) was split # correctly at about 50% validation_size_without_none = len(data_set.validation_data) - num_none_intent_sentences assert abs(len(data_set.training_data) - validation_size_without_none) <= 1 # Verify that all entries were loaded assert len(data_set.training_data) + len(data_set.validation_data) == size # Make sure that the data returned on subsequent calls is the same assert data_set.training_data == data_set.training_data assert data_set.validation_data == data_set.validation_data # Verify that the data is sorted and split differently after reshuffling the data training_data = data_set.training_data validation_data = data_set.validation_data data_set.reshuffle() assert training_data != data_set.training_data assert validation_data != data_set.validation_data # Make sure that a copy of the data is returned and not a reference data_set.training_data.pop() data_set.validation_data.pop() assert len(data_set.training_data) + len(data_set.validation_data) == size
def langcodes_score(language, segment, score): '''Use langcodes on selected URL segments and integrate them into a score.''' # see also: https://babel.pocoo.org/en/latest/locale.html # test if the code looks like a country or a language if segment[:2] not in COUNTRY_CODES and segment[:2] not in LANGUAGE_CODES: return score # test if tag is valid (caution: private codes are) if tag_is_valid(segment): # try to identify language code identified = Language.get(segment).language # see if it matches if identified is not None: LOGGER.debug('langcode %s found in URL segment %s', identified, segment) if identified != language: score -= 1 else: score += 1 return score
def _prepare_user_data(scenario_user): lang_code = random.choice(['ko', 'en']) language = Language.get(lang_code) user_id = random_string()[0:10] default_user = { 'user_id': user_id, 'password': user_id, 'name': 'Steven' + random_string()[0:5], 'language': language.__str__(), 'timezone': 'Asia/Seoul', 'tags': { 'aa': 'bb' }, 'email': 'Steven' + random_string()[0:5] + '@mz.co.kr', 'mobile': '+821026671234' } # Overwrite param, if needed default_user.update(scenario_user) return default_user
def get_audio_print(self, audio: List[Track]) -> List[str]: if not audio: return ["--"] data = [] for t in audio: if t.title and "Commentary" in t.title: title = t.title else: title = Language.get(t.language).display_name() if t.channel_layout: channels = float( sum( self.AUDIO_CHANNEL_LAYOUT_WEIGHT.get(x, 1) for x in t.channel_layout.split(" "))) else: channels = float(t.channel_s) bit_rate_mode = f" ({t.bit_rate_mode})" if t.bit_rate_mode else "" l1 = f"- {title}, {t.format} {channels} @ {t.other_bit_rate[0]}{bit_rate_mode}" data += [(" " + x if i > 0 else x) for i, x in enumerate(textwrap.wrap(l1, 64))] return data
def test_create_owner(self): lang_code = random.choice(['zh-hans', 'jp', 'ko', 'en', 'es']) language = Language.get(lang_code) owner_id = utils.random_string() params = { 'owner_id': owner_id, 'password': utils.generate_password(), 'name': 'Steven' + utils.random_string(), 'language': language.__str__(), 'timezone': 'Asia/Seoul', 'email': 'Steven' + utils.random_string() + '@mz.co.kr', 'domain_id': self.domain.domain_id } owner = self.identity_v1.DomainOwner.create(params) self.domain_owner = owner self.params = params self.assertEqual(params['name'], self.domain_owner.name) self._issue_owner_token(params['owner_id'], params['password'])
def _test_create_user(self, name='test', user_id=None): if self.role is None: self._test_create_role() if user_id is None: user_id = utils.random_string()[0:10] lang_code = random.choice(['zh-hans', 'jp', 'ko', 'en', 'es']) language = Language.get(lang_code) params = { 'user_id': user_id, 'domain_id': self.domain.domain_id, 'password': '******', 'name': name + utils.random_string()[0:5], 'language': language.__str__(), 'timezone': 'utc+9', 'tags': { 'aa': 'bb' }, 'email': name + utils.random_string()[0:5] + '@mz.co.kr', 'mobile': '+821026671234', 'group': 'group-id' } self.user = self.identity_v1.User.create(params, metadata=(('token', self.token), )) self.user = self.identity_v1.User.update_role( { 'user_id': self.user.user_id, 'domain_id': self.domain.domain_id, 'roles': [self.role.role_id] }, metadata=(('token', self.token), )) self.users.append(self.user) return self.user
def test_create_owner(self): lang_code = random.choice(['zh-hans', 'jp', 'ko', 'en', 'es']) language = Language.get(lang_code) owner_id = utils.random_string()[0:10] param = { 'owner_id': owner_id, 'password': '******', 'name': 'Steven' + utils.random_string()[0:5], 'language': language.__str__(), 'timezone': 'utc+9', 'email': 'Steven' + utils.random_string()[0:5] + '@mz.co.kr', 'mobile': '+821026671234', 'domain_id': self.domain.domain_id } owner = self.identity_v1.DomainOwner.create( param ) self.domain_owner = owner self.param = param self.assertEqual(param['name'], self.domain_owner.name)
def __init__(self, language): if isinstance(language, Language): self.language = language else: self.language = Language.get(language) self.token = None
def get_language_info(language): """ Looks up the things we need to know about how to handle text in a given language. This will return a dictionary with the following fields: 'script': a BCP 47 script code such as 'Latn', 'Cyrl', 'Hans'... Indicates the script that tokens in this language should be in, _after_ our preprocessing. The script for 'zh' is 'Hans', for example, because even if the input is in Traditional Chinese ('Hant'), we convert it to Simplified. 'tokenizer': 'regex', 'jieba', 'mecab', or None Indicates the best way we know to separate tokens in the language. 'regex' is what will be used for most languages, meaning that we can segment the text with a Unicode-aware regular expression. If a language generally uses spaces to separate words, the regex will work well. 'jieba' and 'mecab' are tokenizers for specific languages written without spaces. A tokenizer of None means we don't have a good way to segment the language. We'll use the regex anyway, but the results will be pretty bad. 'normal_form': 'NFC' or 'NFKC' How "should" Unicode be normalized when comparing text in this language? This is not a standard, it's just based on experience. Many languages need NFKC normalization for text comparisons to work properly, but in many European languages, NFKC normalization is excessive and loses information. 'remove_marks': True or False Determines whether marks and decorations, such as vowel points and tatweels, should be removed. True for languages in abjad scripts. 'dotless_i': True or False Is "ı" the lowercase of "I" in this language, as in Turkish? 'diacritics_under': 'cedillas', 'commas', or None Should we convert any diacritics that are under the letters "s" and "t" in this language? 'cedillas' means we should convert commas to cedillas, and 'commas' means we should convert cedillas to commas. 'transliteration': 'sr-Latn', 'az-Latn', or None Indicates a type of transliteration that we should use for normalizing a multi-script language. 'sr-Latn' means to use Serbian romanization, and 'az-Latn' means to use Azerbaijani romanization. 'lookup_transliteration': 'zh-Hans' or None Indicates a lossy transliteration that should be not be used for output, but should be applied when looking up words in a list. 'zh-Hans' means that we should convert Traditional Chinese characters to Simplified. """ # The input is probably a string, so parse it into a Language. If it's # already a Language, it will pass through. language = Language.get(language) # Assume additional things about the language, such as what script it's in, # using the "likely subtags" table language_full = language.maximize() # Start the `info` dictionary with default values, including the 'script' # value that we now know from `language_full`. info = { 'script': language_full.script, 'tokenizer': 'regex', 'normal_form': 'NFKC', 'remove_marks': False, 'dotless_i': False, 'diacritics_under': None, 'transliteration': None, 'lookup_transliteration': None } if _language_in_list(language, ['ja', 'ko']): info['tokenizer'] = 'mecab' elif _language_in_list(language, ['zh', 'yue']): info['tokenizer'] = 'jieba' elif info['script'] in SPACELESS_SCRIPTS: info['tokenizer'] = None # Cased alphabetic scripts get NFC normal form if info['script'] in ['Latn', 'Grek', 'Cyrl']: info['normal_form'] = 'NFC' if info['script'] in ['Arab', 'Hebr']: info['remove_marks'] = True if _language_in_list(language, ['tr', 'az', 'kk']): info['dotless_i'] = True info['diacritics_under'] = 'cedillas' elif _language_in_list(language, ['ro']): info['diacritics_under'] = 'commas' if _language_in_list(language, ['sr']): info['transliteration'] = 'sr-Latn' elif _language_in_list(language, ['az']): info['transliteration'] = 'az-Latn' if language.language == 'zh' and language.script != 'Hant': info['lookup_transliteration'] = 'zh-Hans' return info
def language_display(language, display_language=None): if type(language) is str: language = Language.get(language) if display_language is None: return language.display_name() return language.display_name(display_language)
def get_language_info(language): """ Looks up the things we need to know about how to handle text in a given language. This will return a dictionary with the following fields: 'script': a BCP 47 script code such as 'Latn', 'Cyrl', 'Hans'... Indicates the script that tokens in this language should be in, _after_ our preprocessing. The script for 'zh' is 'Hans', for example, because even if the input is in Traditional Chinese ('Hant'), we convert it to Simplified. 'tokenizer': 'regex', 'jieba', 'mecab', or None Indicates the best way we know to separate tokens in the language. 'regex' is what will be used for most languages, meaning that we can segment the text with a Unicode-aware regular expression. If a language generally uses spaces to separate words, the regex will work well. 'jieba' and 'mecab' are tokenizers for specific languages written without spaces. A tokenizer of None means we don't have a good way to segment the language. We'll use the regex anyway, but the results will be pretty bad. 'normal_form': 'NFC' or 'NFKC' How "should" Unicode be normalized when comparing text in this language? This is not a standard, it's just based on experience. Many languages need NFKC normalization for text comparisons to work properly, but in many European languages, NFKC normalization is excessive and loses information. 'remove_marks': True or False Determines whether marks and decorations, such as vowel points and tatweels, should be removed. True for languages in abjad scripts. 'dotless_i': True or False Is "ı" the lowercase of "I" in this language, as in Turkish? 'diacritics_under': 'cedillas', 'commas', or None Should we convert any diacritics that are under the letters "s" and "t" in this language? 'cedillas' means we should convert commas to cedillas, and 'commas' means we should convert cedillas to commas. 'transliteration': 'sr-Latn', 'az-Latn', or None Indicates a type of transliteration that we should use for normalizing a multi-script language. 'sr-Latn' means to use Serbian romanization, and 'az-Latn' means to use Azerbaijani romanization. 'lookup_transliteration': 'zh-Hans' or None Indicates a lossy transliteration that should be not be used for output, but should be applied when looking up words in a list. 'zh-Hans' means that we should convert Traditional Chinese characters to Simplified. """ # The input is probably a string, so parse it into a Language. If it's # already a Language, it will pass through. language = Language.get(language) # Assume additional things about the language, such as what script it's in, # using the "likely subtags" table language_full = language.maximize() # Start the `info` dictionary with default values, including the 'script' # value that we now know from `language_full`. info = { 'script': language_full.script, 'tokenizer': 'regex', 'normal_form': 'NFKC', 'remove_marks': False, 'dotless_i': False, 'diacritics_under': None, 'transliteration': None, 'lookup_transliteration': None } if _language_in_list(language, ['ja', 'ko']): info['tokenizer'] = 'mecab' elif _language_in_list(language, ['zh', 'yue']): info['tokenizer'] = 'jieba' elif info['script'] in SPACELESS_SCRIPTS: info['tokenizer'] = None # Cased alphabetic scripts get NFC normal form if info['script'] in ['Latn', 'Grek', 'Cyrl']: info['normal_form'] = 'NFC' if info['script'] in ['Arab', 'Hebr']: info['remove_marks'] = True if _language_in_list(language, ['tr', 'az', 'kk']): info['dotless_i'] = True info['diacritics_under'] = 'cedillas' elif _language_in_list(language, ['ro']): info['diacritics_under'] = 'commas' if _language_in_list(language, ['sr']): info['transliteration'] = 'sr-Latn' elif _language_in_list(language, ['az']): info['transliteration'] = 'az-Latn' if language.language == 'zh' and language.script != 'Hant': info['lookup_transliteration'] = 'zh-Hans' return info
for lang in SOUP.find_all('code'): curlang = str(lang.getText()) if curlang == 'en' or curlang == 'la' or curlang == 'ceb': continue if curlang == 'zh-CN' or curlang == 'zh-TW': continue filename = Path(curlang + ".po") if filename.is_file(): print(filename, " already exists, skipping...") continue else: print(filename, " doesn't exist, processing...") language = Language.get(curlang).language_name('en') POT_DATA.metadata['Language-Team'] = language + ' <' + curlang + '@li.org>' POT_DATA.metadata['Language'] = curlang for i in range(len(POT_DATA)): tmpstr = POT_DATA[i].msgid.replace('%1', '__') tmpTrans = TRANS.translate(tmpstr, source_language='en', target_language=curlang)['translatedText'] POT_DATA[i].msgstr = tmpTrans POT_DATA[i].msgstr = POT_DATA[i].msgstr.replace('__', '%1') F2 = open(filename, 'wt') F2.write(str(POT_DATA)) F2.close()
def get_language_name(chars): """ """ name = Language.get(chars).language_name() return name
def cache_IETF(): cache = {} result = query( """ select expr.txt, exprsrc.txt as trans_txt from expr inner join denotationx as denotation on denotation.expr = expr.id inner join denotationx as denotationsrc on denotationsrc.meaning = denotation.meaning and denotationsrc.expr != denotation.expr inner join expr as exprsrc on exprsrc.id = denotationsrc.expr where expr.langvar = uid_langvar('art-274') and denotationsrc.langvar = uid_langvar('art-420') """) for r in result: cache[r['txt']] = {} cache[r['txt']]['IETF'] = set() for r in query( """ select langvar.lang_code, langvar.region_expr, uid(langvar.lang_code,langvar.var_code), script_expr.txt as script_expr_txt, uid(region_expr_langvar.lang_code,region_expr_langvar.var_code) as region_expr_uid, region_expr.txt as region_expr_txt from langvar inner join expr on expr.id = langvar.name_expr inner join expr as script_expr on script_expr.id = langvar.script_expr inner join expr as region_expr on region_expr.id = langvar.region_expr inner join langvar as region_expr_langvar on region_expr_langvar.id = region_expr.langvar where uid(langvar.lang_code,langvar.var_code) = any(%s) """, (list(cache.keys()),)): cache[r['uid']].update(r) regions_dict = {} for r in query( """ select expr.txt, exprsrc.txt as trans_txt from expr inner join denotationx as denotation on denotation.expr = expr.id inner join denotationx as denotationsrc on denotationsrc.meaning = denotation.meaning and denotationsrc.expr != denotation.expr inner join expr as exprsrc on exprsrc.id = denotationsrc.expr where expr.langvar = uid_langvar('art-006') and denotationsrc.expr = any(%s) """, ([l['region_expr'] for l in cache.values()],)): if len(r['txt']) == 2: regions_dict[r['trans_txt']] = r['txt'] for r in result: uid = r['txt'] lang = cache[uid] given_tag = Language.get(r['trans_txt'], normalize=False) normalized_tag = Language.get(r['trans_txt'], normalize=True) language_set = {lang['lang_code'], given_tag.language, normalized_tag.language} script_set = {lang['script_expr_txt'], given_tag.script, normalized_tag.script} region_set = {given_tag.region, normalized_tag.region} if lang['region_expr_uid'] == 'art-279' and lang['region_expr_txt'] != '001': region_set.add(lang['region_expr_txt']) try: region_set.add(regions_dict[lang['region_expr_txt']]) except KeyError: pass if {'GB', 'UK'} & region_set: region_set |= {'GB', 'UK'} if {'001', None} & region_set: region_set |= {'001', None} for language, script, region in product(language_set, script_set, region_set): new_tag = normalized_tag.update_dict({'language': language, 'script': script, 'region': region}) cache[uid]['IETF'].add(str(new_tag)) if lang['region_expr_uid'] == 'art-279' and lang['region_expr_txt'] == '001': for language, script, region in product({lang['lang_code']}, script_set, {'001', None}): new_tag = normalized_tag.update_dict({'language': language, 'script': script, 'region': region}) cache[uid]['IETF'].add(str(new_tag)) return cache
def resolve_language(context, language_tag): lang = Language.get(language_tag) request_language = get_language_from_request(context['request']) return lang.language_name(request_language)
def json_export(out: Export = Export("sbx_metadata/[metadata.id].json"), corpus_id: Corpus = Corpus(), lang: Language = Language(), metadata: dict = Config("metadata"), sentences: AnnotationCommonData = AnnotationCommonData( "misc.<sentence>_count"), tokens: AnnotationCommonData = AnnotationCommonData( "misc.<token>_count"), korp_protected: bool = Config("korp.protected"), korp_mode: bool = Config("korp.mode"), md_trainingdata: bool = Config("sbx_metadata.trainingdata"), md_xml_export: str = Config("sbx_metadata.xml_export"), md_stats_export: bool = Config("sbx_metadata.stats_export"), md_korp: bool = Config("sbx_metadata.korp"), md_downloads: list = Config("sbx_metadata.downloads"), md_interface: list = Config("sbx_metadata.interface"), md_contact: dict = Config("sbx_metadata.contact_info")): """Export corpus metadata to JSON format.""" md_obj = {} md_obj["id"] = corpus_id md_obj["type"] = "corpus" md_obj["trainingdata"] = md_trainingdata # Set language info md_obj["lang"] = [{ "code": lang, "name_en": languages.get(part3=lang).name if lang in languages.part3 else lang, "name_sv": Language.get(lang).display_name("swe"), }] # Set name and description md_obj["name_en"] = metadata.get("name", {}).get("eng") md_obj["name_sv"] = metadata.get("name", {}).get("swe") md_obj["description_en"] = metadata.get("description", {}).get("eng") md_obj["description_sv"] = metadata.get("description", {}).get("swe") # Set downloads downloads = [] downloads.append( metadata_utils.make_standard_xml_export(md_xml_export, corpus_id)) downloads.append( metadata_utils.make_standard_stats_export(md_stats_export, corpus_id)) downloads.append(metadata_utils.make_metashare(corpus_id)) downloads.extend(md_downloads) md_obj["downloads"] = [d for d in downloads if d] # Set interface interface = [] interface.append(metadata_utils.make_korp(md_korp, corpus_id, korp_mode)) interface.extend(md_interface) md_obj["interface"] = [d for d in interface if d] # Set contact info if md_contact == "sbx-default": md_obj["contact_info"] = metadata_utils.SBX_DEFAULT_CONTACT else: md_obj["contact_info"] = md_contact # Set size md_obj["size"] = {"tokens": tokens.read(), "sentences": sentences.read()} # Write JSON to file os.makedirs(os.path.dirname(out), exist_ok=True) json_str = json.dumps(md_obj, ensure_ascii=False, indent=4) with open(out, "w") as f: f.write(json_str) logger.info("Exported: %s", out)