def available_isi_penting():
"""
List available transformer models for isi penting generator.
"""
from malaya.function import describe_availability
return describe_availability(_isi_penting_availability)
def available_transformer():
"""
List available transformer zero-shot models.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_availability)
def describe():
"""
Describe Entities supported.
"""
d = [
{'Tag': 'OTHER', 'Description': 'other'},
{
'Tag': 'law',
'Description': 'law, regulation, related law documents, documents, etc',
},
{'Tag': 'location', 'Description': 'location, place'},
{
'Tag': 'organization',
'Description': 'organization, company, government, facilities, etc',
},
{
'Tag': 'person',
'Description': 'person, group of people, believes, unique arts (eg; food, drink), etc',
},
{'Tag': 'quantity', 'Description': 'numbers, quantity'},
{'Tag': 'time', 'Description': 'date, day, time, etc'},
{'Tag': 'event', 'Description': 'unique event happened, etc'},
]
from malaya.function import describe_availability
return describe_availability(d, transpose = False)
def available_transformer_standard_language():
"""
List available transformer models.
"""
from malaya.function import describe_availability
return describe_availability(_standard_transformer_availability)
def available_vectorizer():
"""
List available vectorizer topic modeling.
"""
from malaya.function import describe_availability
return describe_availability(_vectorizer_availability)
def describe():
"""
Describe Part-Of-Speech supported.
"""
d = [
{'Tag': 'ADJ', 'Description': 'Adjective, kata sifat'},
{'Tag': 'ADP', 'Description': 'Adposition'},
{'Tag': 'ADV', 'Description': 'Adverb, kata keterangan'},
{'Tag': 'ADX', 'Description': 'Auxiliary verb, kata kerja tambahan'},
{'Tag': 'CCONJ', 'Description': 'Coordinating conjuction, kata hubung'},
{'Tag': 'DET', 'Description': 'Determiner, kata penentu'},
{'Tag': 'NOUN', 'Description': ' Noun, kata nama'},
{'Tag': 'NUM', 'Description': 'Number, nombor'},
{'Tag': 'PART', 'Description': 'Particle'},
{'Tag': 'PRON', 'Description': 'Pronoun, kata ganti'},
{'Tag': 'PROPN', 'Description': 'Proper noun, kata ganti nama khas'},
{'Tag': 'SCONJ', 'Description': 'Subordinating conjunction'},
{'Tag': 'SYM', 'Description': 'Symbol'},
{'Tag': 'VERB', 'Description': 'Verb, kata kerja'},
{'Tag': 'X', 'Description': 'Other'},
]
from malaya.function import describe_availability
return describe_availability(d, transpose=False)
def available_vectorizer():
"""
List available vectorizer for malaya.summarization.extractive.lda and malaya.summarization.extractive.lsa .
"""
from malaya.function import describe_availability
return describe_availability(_vectorizer_availability)
def available_skipthought():
"""
List available deep skip-thought models.
"""
from malaya.function import describe_availability
return describe_availability(_skipthought_availability)
def available_gpt2():
"""
List available gpt2 generator models.
"""
from malaya.function import describe_availability
return describe_availability(_gpt2_availability)
def available_wordvector():
"""
List available transformer models.
"""
from malaya.function import describe_availability
return describe_availability(_wordvector_availability)
def available_t5():
"""
List available T5 models.
"""
from malaya.function import describe_availability
return describe_availability(_t5_availability)
def available_transformer():
"""
List available transformer models.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_availability,
text='tested on 100k MY-EN sentences.')
def available_transformer():
"""
List available transformer keyword similarity model.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_availability,
text='tested on 20% test set.')
def available_transformer():
"""
List available transformer Entity Tagging models.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_availability,
text='tested on 20% test set.')
def available_transformer_squad():
"""
List available Transformer Span models.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_squad_availability,
text='tested on SQUAD V2 Dev set.')
def available_transformer_ontonotes5():
"""
List available transformer Entity Tagging models trained on Ontonotes 5 Bahasa.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_ontonotes5_availability,
text='tested on 20% test set.')
def available_transformer():
"""
List available transformer models.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_availability,
text='tested on 1k paraphrase texts.')
def available_transformer():
"""
List available transformer subjective analysis models.
"""
from malaya.function import describe_availability
return describe_availability(_transformer_availability,
text='tested on 20% test set.')
def available_transformer():
"""
List available transformer models.
"""
from malaya.function import describe_availability
return describe_availability(
_transformer_availability,
text='tested on 12k CNN + DailyNews test set.')
def describe_ontonotes5():
"""
Describe OntoNotes5 Entities supported. https://spacy.io/api/annotation#named-entities
"""
d = [
{'Tag': 'OTHER', 'Description': 'other'},
{'Tag': 'ADDRESS', 'Description': 'Address of physical location.'},
{'Tag': 'PERSON', 'Description': 'People, including fictional.'},
{
'Tag': 'NORP',
'Description': 'Nationalities or religious or political groups.',
},
{
'Tag': 'FAC',
'Description': 'Buildings, airports, highways, bridges, etc.',
},
{
'Tag': 'ORG',
'Description': 'Companies, agencies, institutions, etc.',
},
{'Tag': 'GPE', 'Description': 'Countries, cities, states.'},
{
'Tag': 'LOC',
'Description': 'Non-GPE locations, mountain ranges, bodies of water.',
},
{
'Tag': 'PRODUCT',
'Description': 'Objects, vehicles, foods, etc. (Not services.)',
},
{
'Tag': 'EVENT',
'Description': 'Named hurricanes, battles, wars, sports events, etc.',
},
{'Tag': 'WORK_OF_ART', 'Description': 'Titles of books, songs, etc.'},
{'Tag': 'LAW', 'Description': 'Named documents made into laws.'},
{'Tag': 'LANGUAGE', 'Description': 'Any named language.'},
{
'Tag': 'DATE',
'Description': 'Absolute or relative dates or periods.',
},
{'Tag': 'TIME', 'Description': 'Times smaller than a day.'},
{'Tag': 'PERCENT', 'Description': 'Percentage, including "%".'},
{'Tag': 'MONEY', 'Description': 'Monetary values, including unit.'},
{
'Tag': 'QUANTITY',
'Description': 'Measurements, as of weight or distance.',
},
{'Tag': 'ORDINAL', 'Description': '"first", "second", etc.'},
{
'Tag': 'CARDINAL',
'Description': 'Numerals that do not fall under another type.',
},
]
from malaya.function import describe_availability
return describe_availability(d, transpose = False)
def available_t5():
"""
List available T5 models.
"""
from malaya.function import describe_availability
return describe_availability(_t5_availability,
text='tested on 5k CNN test set.')
def available_gpt2():
"""
List available gpt2 generator models.
"""
from malaya.function import describe_availability
return describe_availability(
_gpt2_availability,
text='calculate perplexity on never seen malay karangan.')
def available_transformer():
"""
List available transformer models.
"""
from malaya.function import describe_availability
return describe_availability(
_transformer_availability,
text='tested on 7.5k kesalahan tatabahasa texts.',
)
def available_transformer(version: str = 'v2'):
"""
List available transformer dependency parsing models.
Parameters
----------
version : str, optional (default='v2')
Version supported. Allowed values:
* ``'v1'`` - version 1, maintain for knowledge graph.
* ``'v2'`` - Trained on bigger dataset, better version.
"""
from malaya.function import describe_availability
return describe_availability(
_transformer_availability[_validate_version(version)],
text='tested on 20% test set.')
def available_aggregate_function():
from malaya.function import describe_availability
return describe_availability(_aggregate_availability)
def describe():
"""
Describe Dependency supported.
"""
d = [
{
'Tag': 'acl',
'Description': 'clausal modifier of noun'
},
{
'Tag': 'advcl',
'Description': 'adverbial clause modifier'
},
{
'Tag': 'advmod',
'Description': 'adverbial modifier'
},
{
'Tag': 'amod',
'Description': 'adjectival modifier'
},
{
'Tag': 'appos',
'Description': 'appositional modifier'
},
{
'Tag': 'aux',
'Description': 'auxiliary'
},
{
'Tag': 'case',
'Description': 'case marking'
},
{
'Tag': 'ccomp',
'Description': 'clausal complement'
},
{
'Tag': 'compound',
'Description': 'compound'
},
{
'Tag': 'compound:plur',
'Description': 'plural compound'
},
{
'Tag': 'conj',
'Description': 'conjunct'
},
{
'Tag': 'cop',
'Description': 'cop'
},
{
'Tag': 'csubj',
'Description': 'clausal subject'
},
{
'Tag': 'dep',
'Description': 'dependent'
},
{
'Tag': 'det',
'Description': 'determiner'
},
{
'Tag': 'fixed',
'Description': 'multi-word expression'
},
{
'Tag': 'flat',
'Description': 'name'
},
{
'Tag': 'iobj',
'Description': 'indirect object'
},
{
'Tag': 'mark',
'Description': 'marker'
},
{
'Tag': 'nmod',
'Description': 'nominal modifier'
},
{
'Tag': 'nsubj',
'Description': 'nominal subject'
},
{
'Tag': 'obj',
'Description': 'direct object'
},
{
'Tag': 'parataxis',
'Description': 'parataxis'
},
{
'Tag': 'root',
'Description': 'root'
},
{
'Tag': 'xcomp',
'Description': 'open clausal complement'
},
]
from malaya.function import describe_availability
return describe_availability(
d,
transpose=False,
text=
'you can read more from https://universaldependencies.org/treebanks/id_pud/index.html',
)
def describe_tagging():
"""
Describe kesalahan tatabahasa supported.
Full description at https://tatabahasabm.tripod.com/tata/salahtata.htm
"""
d = [
{
'class': 0,
'Description': 'PAD',
'salah': '',
'betul': ''
},
{
'class': 1,
'Description': 'kesambungan subwords',
'salah': '',
'betul': '',
},
{
'class': 2,
'Description': 'tiada kesalahan',
'salah': '',
'betul': '',
},
{
'class': 3,
'Description':
'kesalahan frasa nama, Perkara yang diterangkan mesti mendahului "penerang"',
'salah': 'Cili sos',
'betul': 'sos cili',
},
{
'class': 4,
'Description': 'kesalahan kata jamak',
'salah': 'mereka-mereka',
'betul': 'mereka',
},
{
'class': 5,
'Description': 'kesalahan kata penguat',
'salah': 'sangat tinggi sekali',
'betul': 'sangat tinggi',
},
{
'class': 6,
'Description': 'kata adjektif dan imbuhan "ter" tanpa penguat.',
'salah': 'Sani mendapat markah yang tertinggi sekali.',
'betul': 'Sani mendapat markah yang tertinggi.',
},
{
'class': 7,
'Description': 'kesalahan kata hubung',
'salah': 'Sally sedang membaca bila saya tiba di rumahnya.',
'betul': 'Sally sedang membaca apabila saya tiba di rumahnya.',
},
{
'class': 8,
'Description': 'kesalahan kata bilangan',
'salah': 'Beribu peniaga tidak membayar cukai pendapatan.',
'betul': 'Beribu-ribu peniaga tidak membayar cukai pendapatan',
},
{
'class': 9,
'Description': 'kesalahan kata sendi',
'salah': 'Umar telah berpindah daripada sekolah ini bulan lalu.',
'betul': 'Umar telah berpindah dari sekolah ini bulan lalu.',
},
{
'class': 10,
'Description': 'kesalahan penjodoh bilangan',
'salah': 'Setiap orang pelajar',
'betul': 'Setiap pelajar.',
},
{
'class': 11,
'Description': 'kesalahan kata ganti diri',
'salah':
'Pencuri itu telah ditangkap. Beliau dibawa ke balai polis.',
'betul': 'Pencuri itu telah ditangkap. Dia dibawa ke balai polis.',
},
{
'class': 12,
'Description': 'kesalahan ayat pasif',
'salah': 'Cerpen itu telah dikarang oleh saya.',
'betul': 'Cerpen itu telah saya karang.',
},
{
'class': 13,
'Description': 'kesalahan kata tanya',
'salah': 'Kamu berasal dari manakah ?',
'betul': 'Kamu berasal dari mana ?',
},
{
'class': 14,
'Description': 'kesalahan tanda baca',
'salah': 'Kamu berasal dari manakah .',
'betul': 'Kamu berasal dari mana ?',
},
{
'class': 15,
'Description': 'kesalahan kata kerja tak transitif',
'salah': 'Dia kata kepada saya',
'betul': 'Dia berkata kepada saya',
},
{
'class': 16,
'Description': 'kesalahan kata kerja transitif',
'salah': 'Dia suka baca buku',
'betul': 'Dia suka membaca buku',
},
{
'class': 17,
'Description': 'penggunaan kata yang tidak tepat',
'salah': 'Tembuk Besar negeri Cina dibina oleh Shih Huang Ti.',
'betul': 'Tembok Besar negeri Cina dibina oleh Shih Huang Ti',
},
]
from malaya.function import describe_availability
return describe_availability(d, transpose=False)
