def test_get_tag(self): tag = tags.tag('en') self.assertIsNotNone(tag) tag = tags.tag('en-gb') self.assertIsNotNone(tag) self.assertEqual(tag.format, 'en-GB')
def label(labels=[], language='any'): ''' Provide a label for a list of labels. The items in the list of labels are assumed to be either instances of :class:`Label`, or dicts with at least the key `label` in them. These will be passed to the :func:`dict_to_label` function. This method tries to find a label by looking if there's a pref label for the specified language. If there's no pref label, it looks for an alt label. It disregards hidden labels. While matching languages, preference will be given to exact matches. But, if no exact match is present, an inexact match will be attempted. This might be because a label in language `nl-BE` is being requested, but only `nl` or even `nl-NL` is present. Similarly, when requesting `nl`, a label with language `nl-NL` or even `nl-Latn-NL` will also be considered, providing no label is present that has an exact match with the requested language. If language 'any' was specified, all labels will be considered, regardless of language. To find a label without a specified language, pass `None` as language. If a language or None was specified, and no label could be found, this method will automatically try to find a label in some other language. Finally, if no label could be found, None is returned. :param string language: The preferred language to receive the label in. This should be a valid IANA language tag. ''' # Normalise the tag broader_language_tag = None if language != 'any': language = tags.tag(language).format broader_language_tag = tags.tag(language).language pref = None alt = None for l in labels: l = dict_to_label(l) if language == 'any' or l.language == language: if l.type == 'prefLabel' and (pref is None or pref.language != language): pref = l if l.type == 'altLabel' and (alt is None or alt.language != language): alt = l if broader_language_tag and tags.tag(l.language).language and tags.tag( l.language).language.format == broader_language_tag.format: if l.type == 'prefLabel' and pref is None: pref = l if l.type == 'altLabel' and alt is None: alt = l if pref is not None: return pref elif alt is not None: return alt return label(labels, 'any') if language != 'any' else None
def label(labels=[], language='any'): ''' Provide a label for a list of labels. The items in the list of labels are assumed to be instances of :class:`Label`. This method tries to find a label by looking if there's a pref label for the specified language. If there's no pref label, it looks for an alt label. It disregards hidden labels. While matching languages, preference will be given to exact matches. But, if no exact match is present, an inexact match will be attempted. This might be because a label in language `nl-BE` is being requested, but only `nl` or even `nl-NL` is present. Similarly, when requesting `nl`, a label with language `nl-NL` or even `nl-Latn-NL` will also be considered, providing no label is present that has an exact match with the requested language. If language 'any' was specified, all labels will be considered, regardless of language. To find a label without a specified language, pass `None` as language. If a language or None was specified, and no label could be found, this method will automatically try to find a label in some other language. Finally, if no label could be found, None is returned. :param list labels: A list of :class:`labels <Label>`. :param str language: The language for which a label should preferentially be returned. This should be a valid IANA language tag. :rtype: A :class:`Label` or `None` if no label could be found. ''' # Normalise the tag broader_language_tag = None if language != 'any': language = tags.tag(language).format broader_language_tag = tags.tag(language).language pref = None alt = None for l in labels: labeltype = l.labeltype_id or l.labeltype.name if language == 'any' or l.language_id == language: if labeltype == 'prefLabel' and (pref is None or pref.language_id != language): pref = l if labeltype == 'altLabel' and (alt is None or alt.language_id != language): alt = l if broader_language_tag and tags.tag(l.language_id).language and tags.tag( l.language_id).language.format == broader_language_tag.format: if labeltype == 'prefLabel' and pref is None: pref = l if labeltype == 'altLabel' and alt is None: alt = l if pref is not None: return pref elif alt is not None: return alt return label(labels, 'any') if language != 'any' else None
def test_redundant_subtags(self): tag = tags.tag('es-419') self.assertListEqual(tag.region.description, ['Latin America and the Caribbean']) self.assertListEqual(tag.language.description, ['Spanish', 'Castilian']) tag = tags.tag('sgn-NL') self.assertListEqual(tag.region.description, ['Netherlands']) self.assertListEqual(tag.language.description, ['Sign languages'])
def parse_hreflang_value(self, hreflang_value): """ NBED """ # Replace underscores if desired if self.allow_underscore: hreflang_value = hreflang_value.replace("_", "-") # Handle the x-default special case if hreflang_value.lower() == "x-default": return ("x-default", "default", "default") # Try to parse the IETF language/region tag. parsed_tag = tags.tag(hreflang_value) # Extract the language language = str(parsed_tag.language.description[0].encode('ascii', errors='xmlcharrefreplace') if parsed_tag.language else "Unknown") # Extract the region # Differentiate between none being specified and one not # being recognised region = (str(parsed_tag.region.description[0].encode('ascii', errors='xmlcharrefreplace')) if parsed_tag.region else "Unknown" if len(str(parsed_tag)) > 3 else None) # Return cleaned version of the tag return (str(parsed_tag), language, region)
def _apply(self, value): # Lazy-load dependencies to reduce memory usage when this # filter is not used in a project. from language_tags import tags from language_tags.Tag import Tag value = self._filter(value, Type(string_types + (Tag, ))) if self._has_errors: return None if isinstance(value, Tag): return value tag = tags.tag(value) if not tag.valid: return self._invalid_value( value=value, reason=self.CODE_INVALID, context={ 'parse_errors': [(error.code, error.message) for error in tag.errors], }, ) return tag
def check_update(self, update: object): if not self.auto_locale: return False if not isinstance(update, Update): return False if not update.effective_user or not update.effective_user.language_code: return self.logger.debug("[%s] Update has language %s.", update.update_id, update.effective_user.language_code) if update.effective_user.language_code and update.effective_user.language_code != self.channel.locale: self.channel.locale = update.effective_user.language_code tag = tags.tag(update.effective_user.language_code) if tag.language: locale = tag.language.format if tag.region: locale += "_" + tag.region.format else: locale = update.effective_user.language_code.replace('-', '_') self.logger.info("Updating locale to %s", locale) self.channel.translator = gettext.translation( "efb_telegram_master", resource_filename('efb_telegram_master', 'locale'), languages=[locale, 'C'], fallback=True) return False
def test_print(self): nlbe = tags.tag('nl-Latn-BE') print(nlbe) self.assertEqual(nlbe.__str__(), 'nl-Latn-BE') print(nlbe.language) self.assertEqual(nlbe.language.__str__(), 'nl') print(nlbe.script) self.assertEqual(nlbe.script.__str__(), 'Latn')
def filter_labels_by_language(labels, language, broader=False): ''' Filter a list of labels, leaving only labels of a certain language. :param list labels: A list of :class:`Label`. :param str language: An IANA language string, eg. `nl` or `nl-BE`. :param boolean broader: When true, will also match `nl-BE` when filtering on `nl`. When false, only exact matches are considered. ''' if language == 'any': return labels if broader: language = tags.tag(language).language.format return [l for l in labels if tags.tag(l.language).language.format == language] else: language = tags.tag(language).format return [l for l in labels if tags.tag(l.language).format == language]
def languagetag_isvalid_rule(node, language_tag, errors): """ Check that a languagetag is a valid IANA language tag. """ if not tags.check(language_tag): errors.append( colander.Invalid( node, 'Invalid language tag: %s' % ", ".join( [err.message for err in tags.tag(language_tag).errors])))
def languagetag_isvalid_rule(node, language_tag, errors): """ Check that a languagetag is a valid IANA language tag. """ if not tags.check(language_tag): errors.append(colander.Invalid( node, 'Invalid language tag: %s' % ", ".join([err.message for err in tags.tag(language_tag).errors]) ))
def validate(api_input): errors = {} language_tag = tags.tag(api_input['language']) if not language_tag.valid: errors['language'] = '{} is not a supported language.'.format( api_input['language']) if len(errors) > 0: raise ValidationError('Input payload validation failed', errors=errors)
def get(self): lang = flask.request.args.get('language', default='en') page = flask.request.args.get('page', default=1, type=int) page_size = flask.request.args.get('page-size', default=10, type=int) formatted_lang = tags.tag(lang).format lang_name = tags.description(lang)[0] if tags.check(lang) else None return game_repository.all_v2(formatted_lang, lang_name, page, page_size)
def parse_language_tag(tag_str): """ Parse and validate a language tag. :param str tag_str: A language tag to parse :raise ValueError: If the language tag is invalid. :rtype: language_tags.Tag.Tag :return: A language subtag object """ tag_obj = tags.tag(tag_str) if tag_obj.valid: return tag_obj raise ValueError("Invalid language tag '{!s}'".format(tag_str))
def _get_items(self, service, params, **kwargs): # send request to Heritagedata """ Returns the results of a service method to a :class:`lst` of concepts (and collections). The return :class:`lst` can be empty. :param service (str): service method :returns: A :class:`lst` of concepts (and collections). Each of these is a dict with the following keys: * id: id within the conceptscheme * uri: :term:`uri` of the concept or collection * type: concept or collection * label: A label to represent the concept or collection. """ request = self.service_scheme_uri + "/" + service try: res = self.session.get(request, params=params) except ConnectionError as e: raise ProviderUnavailableException("Request could not be executed - Request: %s - Params: %s" % (request, params)) if res.status_code == 404: raise ProviderUnavailableException("Service not found (status_code 404) - Request: %s - Params: %s" % (request, params)) res.encoding = 'utf-8' result = res.json() d = {} for r in result: uri = r['uri'] label = None if 'label' in r.keys(): label = r['label'] language = None if 'label lang' in r.keys(): language = r['label lang'] property = None if 'property' in r.keys(): property = r['property'] if not service == 'getConceptRelations' or property == str(SKOS.narrower): item = { 'id': _split_uri(uri, 1), 'uri': uri, 'type': 'concept', 'label': label, 'lang': language } if uri not in d: d[uri] = item if tags.tag(d[uri]['lang']).format == tags.tag(self._get_language(**kwargs)).format: pass elif tags.tag(item['lang']).format == tags.tag(self._get_language(**kwargs)).format: d[uri] = item elif tags.tag(item['lang']).language and (tags.tag(item['lang']).language.format == tags.tag(self._get_language(**kwargs)).language.format): d[uri] = item elif tags.tag(item['lang']).format == 'en': d[uri] = item return list(d.values())
def _get_answer(self, query, **kwargs): # send request to getty """ Returns the results of the Sparql query to a :class:`lst` of concepts and collections. The return :class:`lst` can be empty. :param query (str): Sparql query :returns: A :class:`lst` of concepts and collections. Each of these is a dict with the following keys: * id: id within the conceptscheme * uri: :term:`uri` of the concept or collection * type: concept or collection * label: A label to represent the concept or collection. """ request = self.base_url + "sparql.json" try: res = self.session.get(request, params={"query": query}) except ConnectionError as e: raise ProviderUnavailableException("Request could not be executed - Request: %s - Params: %s" % (request, query)) if res.status_code == 404: raise ProviderUnavailableException("Service not found (status_code 404) - Request: %s - Params: %s" % (request, query)) if not res.encoding: res.encoding = 'utf-8' r = res.json() d = {} for result in r["results"]["bindings"]: uri = result["Subject"]["value"] if "Term" in result: label = result["Term"]["value"] else: label = "<not available>" item = { 'id': result["Id"]["value"], 'uri': uri, 'type': result["Type"]["value"].rsplit('#', 1)[1], 'label': label, 'lang': result["Lang"]["value"] } if uri not in d: d[uri] = item if tags.tag(d[uri]['lang']).format == tags.tag(self._get_language(**kwargs)).format: pass elif tags.tag(item['lang']).format == tags.tag(self._get_language(**kwargs)).format: d[uri] = item elif tags.tag(item['lang']).language and (tags.tag(item['lang']).language.format == tags.tag(self._get_language()).language.format): d[uri] = item elif tags.tag(item['lang']).format == tags.tag('en').format: d[uri] = item return list(d.values())
def validate(api_input): errors = {} language_tag = tags.tag(api_input['language']) if not language_tag.valid: errors['language'] = '{} is not a supported language.'.format( api_input['language']) license = api_input['license'] if license.lower() not in license_dict: errors['license'] = '{} is not a valid license.'.format(license) if len(errors) > 0: raise ValidationError('Input payload validation failed', errors=errors)
def label_lang_rule(errors, node, request, labels): for label in labels: language_tag = label['language'] if not tags.check(language_tag): errors.append(colander.Invalid( node['labels'], 'Invalid language tag: %s' % ", ".join([err.message for err in tags.tag(language_tag).errors]) )) else: languages_present = request.db.query(Language).filter_by(id=language_tag).count() if not languages_present: descriptions = ', '.join(tags.description(language_tag)) language_item = Language(id=language_tag, name=descriptions) request.db.add(language_item)
def to_db_structure(api_input): Game.validate(api_input) return { 'game_uuid': api_input.get('game_uuid', str(uuid.uuid4())), 'external_id': api_input['external_id'], 'title': api_input['title'], 'description': api_input['description'], 'language': tags.tag(api_input['language']).format, 'url': api_input['url'], 'license': api_input['license'], 'source': api_input['source'], 'publisher': api_input['publisher'], 'coverimage': api_input['coverimage']['imageId'] }
def get_scripts(self): """ Retrieve the scripts defined in the metadata. :return: The list alpha4 scripts of defined languages. """ scripts = [] for language in self.languages: lang_tag = rfc5646.tag(language) if lang_tag.valid: iso_script_tag = lang_tag.script if iso_script_tag is not None: iso_script = iso_script_tag.format scripts += script_iso15924_to_unicode(iso_script) return scripts
def label_lang_rule(errors, node, request, labels): for label in labels: language_tag = label['language'] if not tags.check(language_tag): errors.append( colander.Invalid( node['labels'], 'Invalid language tag: %s' % ", ".join( [err.message for err in tags.tag(language_tag).errors]))) else: languages_present = request.db.query(Language).filter_by( id=language_tag).count() if not languages_present: descriptions = ', '.join(tags.description(language_tag)) language_item = Language(id=language_tag, name=descriptions) request.db.add(language_item)
def checkLocalesDirNames(localesDirNames, errors): newErrors = [] for dirName in localesDirNames: tagStr = dirName.replace("_", "-") langTag = tags.tag(tagStr) if not langTag.valid: details = [] for tagError in langTag.errors: details.append(tagError.message) newErrors.append({ 'msg': "'{0}' isn't a valid language tag".format(dirName), 'details': details, 'level': "ERROR" }) errors['localesTag'] = newErrors
def to_api_structure(db_output): api_response = { 'game_uuid': db_output['game_uuid'], 'external_id': db_output['external_id'], 'title': db_output['title'], 'description': db_output['description'], 'language': tags.tag(db_output['language']).format, 'url': db_output['url'], 'license': db_output['license'], 'source': db_output['source'], 'publisher': db_output['publisher'] } cover_image_details = GDLConfig.IMAGE_API_CLIENT.metadata_for( db_output['coverimage']) if cover_image_details: api_response['coverimage'] = cover_image_details.as_dict() return api_response
def label_lang_rule(errors, node, languages_manager, labels): """ Checks that languages of a label are valid. Checks that they are valid IANA language tags. If the language tag was not already present in the database, it adds them. """ for label in labels: language_tag = label['language'] if not tags.check(language_tag): errors.append(colander.Invalid( node['labels'], 'Invalid language tag: %s' % ", ".join([err.message for err in tags.tag(language_tag).errors]) )) else: languages_present = languages_manager.count_languages(language_tag) if not languages_present: descriptions = ', '.join(tags.description(language_tag)) language_item = Language(id=language_tag, name=descriptions) languages_manager.save(language_item)
def label_lang_rule(errors, node, languages_manager, labels): """ Checks that languages of a label are valid. Checks that they are valid IANA language tags. If the language tag was not already present in the database, it adds them. """ for label in labels: language_tag = label['language'] if not tags.check(language_tag): errors.append( colander.Invalid( node['labels'], 'Invalid language tag: %s' % ", ".join( [err.message for err in tags.tag(language_tag).errors]))) else: languages_present = languages_manager.count_languages(language_tag) if not languages_present: descriptions = ', '.join(tags.description(language_tag)) language_item = Language(id=language_tag, name=descriptions) languages_manager.save(language_item)
def test_grandfathered_subtags(self): tag = tags.tag('i-klingon') self.assertIsNone(tag.language) self.assertEqual(len(tag.subtags), 0)
def decorated_method_wrapper(self, *args, **kwargs): tag = tags.tag(decorated(self, *args, **kwargs)) return str(tag) if tag.valid else None
def test_undetermined(self): tag = tags.tag('und') self.assertIsNotNone(tag) self.assertEqual(tag.format, 'und')
def test_pass_tag_object(self): """ The incoming value is already a Tag object. """ self.assertFilterPasses(tags.tag('en-cmn-Hant-HK'))
def bcp47_validate_wrapper(self, *args, **kwargs): tag = tags.tag(decorated(self, *args, **kwargs)) return str(tag) if tag.valid else None
def _get_items(self, service, params, **kwargs): # send request to Heritagedata """ Returns the results of a service method to a :class:`lst` of concepts (and collections). The return :class:`lst` can be empty. :param service (str): service method :returns: A :class:`lst` of concepts (and collections). Each of these is a dict with the following keys: * id: id within the conceptscheme * uri: :term:`uri` of the concept or collection * type: concept or collection * label: A label to represent the concept or collection. """ request = self.service_scheme_uri + "/" + service try: res = self.session.get(request, params=params) except ConnectionError as e: raise ProviderUnavailableException( f"Request could not be executed - Request: {request} - Params: {params}" ) if res.status_code == 404: raise ProviderUnavailableException( f"Service not found (status_code 404) - Request: {request} - Params: {params}" ) res.encoding = 'utf-8' result = res.json() d = {} for r in result: uri = r['uri'] label = None if 'label' in r.keys(): label = r['label'] language = None if 'label lang' in r.keys(): language = r['label lang'] property = None if 'property' in r.keys(): property = r['property'] if not service == 'getConceptRelations' or property == "skos:narrower": item = { 'id': _split_uri(uri, 1), 'uri': uri, 'type': 'concept', 'label': label, 'lang': language } if uri not in d: d[uri] = item if tags.tag(d[uri]['lang']).format == tags.tag( self._get_language(**kwargs)).format: pass elif tags.tag(item['lang']).format == tags.tag( self._get_language(**kwargs)).format: d[uri] = item elif tags.tag(item['lang']).language and (tags.tag( item['lang']).language.format == tags.tag( self._get_language(**kwargs)).language.format): d[uri] = item elif tags.tag(item['lang']).format == 'en': d[uri] = item return list(d.values())
def label(labels=[], language='any', sortLabel=False): ''' Provide a label for a list of labels. The items in the list of labels are assumed to be either instances of :class:`Label`, or dicts with at least the key `label` in them. These will be passed to the :func:`dict_to_label` function. This method tries to find a label by looking if there's a pref label for the specified language. If there's no pref label, it looks for an alt label. It disregards hidden labels. While matching languages, preference will be given to exact matches. But, if no exact match is present, an inexact match will be attempted. This might be because a label in language `nl-BE` is being requested, but only `nl` or even `nl-NL` is present. Similarly, when requesting `nl`, a label with language `nl-NL` or even `nl-Latn-NL` will also be considered, providing no label is present that has an exact match with the requested language. It's possible to pass multiple languages as a list. In this case, the method will try handling each language in turn. Please be aware that this includes handling variations. When assing `nl-BE, nl, nl-NL`, the second and third languages will never be handled since handling `nl-BE` includes looking for other related languages such as `nl-NL` and `nl`. If language 'any' was specified, all labels will be considered, regardless of language. To find a label without a specified language, pass `None` as language. If a language or None was specified, and no label could be found, this method will automatically try to find a label in some other language. Finally, if no label could be found, None is returned. ..versionchanged:: 1.1 It is now possible to pass a list of languages. :param any language: The preferred language to receive the label in. This should be a valid IANA language tag or list of language tags. If you pass a list, the order of the languages in the list will be taken into account when trying to determine a label. :param boolean sortLabel: Should sortLabels be considered or not? If True, sortLabels will be preferred over prefLabels. Bear in mind that these are still language dependent. So, it's possible to have a different sortLabel per language. :rtype: A :class:`Label` or `None` if no label could be found. ''' if not labels: return None if isinstance(language, str): language = [language] if isinstance(language, list): language = [l for l in language if tags.tag(l).language] if not language: language = ['und'] labels = [dict_to_label(l) for l in labels] l = False for lang in language: if sortLabel: l = find_best_label_for_type(labels, lang, 'sortLabel') if not l: l = find_best_label_for_type(labels, lang, 'prefLabel') if not l: l = find_best_label_for_type(labels, lang, 'altLabel') if l: return l return label(labels, 'any', sortLabel) if 'any' not in language else None