def parseSolution(s):
def removeComment(y):
if ';' in y: return y[:y.index(';')].strip()
y = y.strip()
if len(y) == 0 or y[0] == '#': return ''
return y
lines = [removeComment(x) for x in s.split('\n')]
prefixes = []
suffixes = []
rules = []
for l in lines:
if 'stem' in l:
[prefix, suffix] = l.split('stem')
prefix = prefix.replace('+', '').strip()
suffix = suffix.replace('+', '').strip()
prefixes.append(Morph(tokenize(prefix)))
suffixes.append(Morph(tokenize(suffix)))
elif len(l) > 0:
r = parseRule(l)
if r == None:
print "Could not parse '%s'" % l
assert False
rules.append(r)
return Solution(rules, prefixes, suffixes)
def finalizeResult(self, k, result):
"""do a final Hail Mary transduction of underlying forms and then expand the frontier"""
if len(result.solutionSequence) == 0:
emptySolution = Solution(prefixes=[Morph(u"")]*self.numberOfInflections,
suffixes=[Morph(u"")]*self.numberOfInflections,
rules=[], underlyingForms={})
result.recordSolution(emptySolution)
return result.lastSolutionIsFinal()
setGlobalTimeout(None)
s = result.solutionSequence[-1][0]
s = self.finalTransduction(s)
f = self.expandFrontier(s, k)
result.recordFinalFrontier(f)
def read_chunks(sentence) -> list:
chunks = [] # 1文の解析結果を格納
chunk = None # 文節
for line in sentence.splitlines():
# 係り受け解析による区切りなら
if line.find('*') == 0:
# 係り先が書かれている位置
dep_par = re.search('[-]*[0-9]+D', line).start()
if chunk is not None:
chunks.append(chunk)
chunk = Chunk([], int(line[dep_par:line.find('D')]), [])
# 1文の終了の記号が見つかれば
elif line.find('EOS') > -1:
if chunk is not None:
chunks.append(chunk)
if len(chunks) > 0:
# 係り受け元の格納
for i in range(len(chunks)):
if chunks[i].dst > -1:
chunks[chunks[i].dst].srcs.append(i)
# 1文を格納
return chunks
else:
# 形態素の情報を文節に格納
speech = line[line.find('\t') + 1:].replace('\n', '').split(',')
morph = Morph(line[0:line.find('\t')], speech[6], speech[0],
speech[1])
chunk.morphs.append(morph)
return chunks_list
def parse():
cabocha = CaboCha.Parser()
result = []
with open('neko.txt') as input_data:
for line in input_data:
line = line.strip()
parsed = cabocha.parse(line).toString(CaboCha.FORMAT_LATTICE)
chunks = {}
for sentence_str in parsed.split('* '):
sentence_analysis = sentence_str.split('\n')
affliation_str = sentence_analysis.pop(0)
if affliation_str in ['', 'EOS']:
continue
morph_analysis = affliation_str.split(' ')
chunk = Chunk()
chunk.id = int(morph_analysis[0])
chunk.srcs = int(morph_analysis[1][:-1])
morphs = []
for morph_str in sentence_analysis:
if morph_str in ['', 'EOS']:
continue
surface, right = morph_str.split('\t')
morph_items = right.split(',')
morphs.append(Morph(surface, morph_items[6],
morph_items[0], morph_items[1]))
chunk.morphs = morphs
chunks[chunk.id] = chunk
for i, chunk in chunks.items():
if chunk.srcs > 0:
chunks[chunk.srcs].dst.append(i)
result.append(chunks)
return result
def dump(node):
debugcnt = 0
sentence_id = -1
for c in node:
if c.tag == "sentence":
sentence = []
sentence_id += 1
chunk_dict = {} # chunkは1sentenceに複数
token_dict = {} #tokenは1chunkに複数
chunk_iter = c.getiterator('chunk')
for chunk in chunk_iter:
morphs = []
tok_iter = chunk.getiterator('tok')
chunk_dict['id'] = chunk.get("id")
chunk_dict['link'] = chunk.get("link")
chunk_dict['rel'] = chunk.get("rel")
chunk_dict['score'] = chunk.get("score")
chunk_dict['head'] = chunk.get("head")
chunk_dict['func'] = chunk.get("func")
for tok in tok_iter:
tok_id = tok.get('id')
tok_feature = tok.get('feature')
morph = Morph(sentence_id, chunk_dict['id'], tok_id, tok_feature, tok.text)
morphs.append(morph)
tok_content = tok.text
a_chunk = Chunk(sentence_id,chunk_dict['id'], chunk_dict['link'], chunk_dict['rel'], \
chunk_dict['score'], chunk_dict['head'], chunk_dict['func'], morphs)
# chuncks.append(a_chunk)
sentence.append(a_chunk)
sents_list.append(sentence)
dump(c)
def __init__(self, data, count, problemName=None):
self.problemName = problemName
self.data = [Morph(tokenize(x)) for x in data]
self.count = count
self.bank = FeatureBank([w for w in data])
self.maximumObservationLength = max([len(w) for w in self.data]) + 1
def __init__(self, morph=None, quality=None):
""" Initialize a Hyphenator class. You may pass in a Morph() instance,
or it can create a new one. If you specify quality, that quality will be
used by default for every call to the hyphenate functions.
"""
self.morph = morph or Morph()
self.default_quality = quality or 2
def main():
# 結果の格納用
morph_list = []
i = 0
with open('neko.txt.cabocha', 'r') as f:
for line in f.readlines():
# 係り受け解析による区切りなら
if line.find('*') == 0:
# 文節の開始地点なら
if int(line[2]) == 0:
# listの中にリストを追加
morph_list.append(list())
# 次のリストの位置を保持
i += 1
# 文章の一部で条件を満たしているなら
#elif line.find('EOS') == -1 and line[0] != ' ':
elif line.find('EOS') == -1:
# 品詞情報を格納
speech = line[line.find('\t') + 1:].replace('\n',
'').split(',')
# 形態素のクラスを現在の文章を指す配列に挿入
morph = Morph(line[0:line.find('\t')], speech[6], speech[0],
speech[1])
morph_list[i - 1].append(morph)
# 3行目を出力
for i in range(len(morph_list[2])):
morph_list[2][i].show()
def __init__(self, data, problemName=None, bank = None, useSyllables = False, UG = None,
fixedMorphology = None):
self.problemName = problemName
self.UG = UG
self.countingProblem = problemName == "Odden_2.4_Tibetan"
if bank != None: self.bank = bank
else:
self.bank = FeatureBank([ w for l in data for w in l if w != None ] + ([u'-'] if useSyllables else []))
self.numberOfInflections = len(data[0])
for d in data: assert len(d) == self.numberOfInflections
# wrap the data in Morph objects if it isn't already
self.data = [ tuple( None if i == None else (i if isinstance(i,Morph) else Morph(tokenize(i)))
for i in Lex)
for Lex in data ]
self.maximumObservationLength = max([ len(w) for l in self.data for w in l if w != None ])
self.wordBoundaries = any([ (u'##' in w.phonemes) for l in self.data for w in l if w ])
# fixedMorphology : list of morphologies, one for each inflection
# Each morphology is either None (don't fix it) or a pair of (prefix, suffix)
if fixedMorphology == None: fixedMorphology = [None]*self.numberOfInflections
self.fixedMorphology = fixedMorphology
assert len(self.fixedMorphology) == self.numberOfInflections
self.inflectionsPerObservation = sum(x is not None
for xs in self.data for x in xs )/len(self.data)
self.pervasiveTimeout = None
self.precomputedAlignment = None
def restrict(self, newData):
restriction = copy.copy(self)
restriction.data = [
tuple(None if i == None else (
i if isinstance(i, Morph) else Morph(tokenize(i)))
for i in Lex) for Lex in newData
]
return restriction
def __init__(self):
self.dictionary = Dictionary()
self.morph = Morph()
self.resp_what = responder.WhatResponder(self.dictionary)
self.resp_random = responder.RandomResponder(self.dictionary)
self.resp_pattern = responder.PatternResponder(self.dictionary)
self.resp_template = responder.TemplateResponder(self.dictionary)
self.responder = self.resp_pattern
def tibetanCountingConstraints(self, stems, prefixes, suffixes):
condition(wordLength(prefixes[0]) == 0)
condition(wordLength(suffixes[0]) == 0)
condition(wordLength(suffixes[1]) == 0)
condition(wordLength(prefixes[2]) == 0)
for n,inflections in enumerate(self.data):
if inflections == (Morph(u"ǰu"),None,None): # 10
condition(wordEqual(stems[n],prefixes[1]))
condition(wordEqual(stems[n],suffixes[2]))
def stochasticSearch(self, iterations, width):
population = [Solution([EMPTYRULE],
[Morph([])]*self.numberOfInflections,
[Morph([])]*self.numberOfInflections)]
for i in range(iterations):
# expand the population
children = [ parent.mutate(self.bank)
for parent in population
for _ in range(width) ]
population += children
populationScores = [ (self.solutionDescriptionLength(s) + s.modelCost(),s)
for s in population ]
populationScores.sort()
population = [ s
for _,s in populationScores[:width] ]
setVerbosity(4)
mdl = self.solutionDescriptionLength(population[0])
setVerbosity(0)
print "MDL:",mdl+population[0].modelCost()
def __init__(self, data, CPUs=1):
self.CPUs = CPUs
self.bank = FeatureBank([w for l in data
for w in l if w != None] + [u'?', u'*'])
self.numberOfInflections = len(data[0])
# wrap the data in Morph objects if it isn't already
self.data = [
tuple(None if i == None else (
i if isinstance(i, Morph) else Morph(tokenize(i)))
for i in Lex) for Lex in data
]
self.maximumObservationLength = max(
[len(w) for l in self.data for w in l if w != None])
def solve(self):
'''
insert your code
'''
with open('neko.txt.cabocha', 'r') as f:
morphologies = []
sentence = []
for line in f.readlines():
if line.strip() == 'EOS':
if len(sentence) > 0:
morphologies.append(sentence)
sentence = []
elif not line.startswith('*'):
surface, result = line.split()
results = result.split(',')
sentence.append(
Morph(surface, results[6], results[0], results[1]))
print morphologies[2]
return None
def read_chunks(filename='') -> list:
chunks_list = [] # 結果の格納用
chunks = [] # 1文の解析結果を格納
chunk = None # 文節
with open(filename, 'r', encoding='utf-8') as f:
for line in f.readlines():
# 係り受け解析による区切りなら
if line.find('*') == 0:
# 係り先が書かれている位置
dep_par = re.search('[-]*[0-9]+D', line).start()
if chunk is not None:
chunks.append(chunk)
chunk = Chunk([], int(line[dep_par:line.find('D')]), [])
# 1文の終了の記号が見つかれば
elif line.find('EOS') > -1:
if chunk is not None:
chunks.append(chunk)
if len(chunks) > 0:
# 係り受け元の格納
for i in range(len(chunks)):
chunks[chunks[i].dst].srcs.append(i)
# 1文を格納
chunks_list.append(chunks)
# リセット
chunk = None
chunks = []
else:
# 形態素の情報を文節に格納
speech = line[line.find('\t') + 1:].replace('\n',
'').split(',')
morph = Morph(line[0:line.find('\t')], speech[6], speech[0],
speech[1])
chunk.morphs.append(morph)
return chunks_list
def morph_model(self, ctx, params):
"""
Morph Function
:param params: instance of type "CallingParams" (Insert your typespec
information here.) -> structure: parameter "fbamodel_workspace" of
String, parameter "fbamodel_id" of String, parameter
"media_workspace" of String, parameter "media_id" of String,
parameter "genome_workspace" of String, parameter "genome_id" of
String, parameter "proteincomparison_workspace" of String,
parameter "proteincomparison_id" of String, parameter "fill_src"
of Long, parameter "translate_media" of Long, parameter
"num_reactions_to_process" of Long, parameter
"translate_media_workspace" of String, parameter
"translate_media_id" of String, parameter "output_id" of String,
parameter "workspace" of String
:returns: instance of type "CallingResults" -> structure: parameter
"report_name" of String, parameter "report_ref" of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN morph_model
self.service = Service(self.callback_url, self.workspaceURL, ctx)
required_args = [
'fbamodel_name', 'fbamodel_workspace', 'media_name',
'media_workspace', 'proteincomparison_name',
'proteincomparison_workspace', 'genome_name', 'genome_workspace',
'output_name'
]
for r in required_args:
if r not in params:
raise ValueError("insufficient params supplied")
def _translate_obj_identity(workspace, name):
info = self.service.get_info(workspace, name=name)
return info[0], workspace
objid, ws = _translate_obj_identity(params['fbamodel_workspace'],
params['fbamodel_name'])
model = FBAModel(objid, ws, service=self.service)
objid, ws = _translate_obj_identity(params['media_workspace'],
params['media_name'])
media = Media(objid, ws, service=self.service)
objid, ws = _translate_obj_identity(
params['proteincomparison_workspace'],
params['proteincomparison_name'])
protcomp = ProteomeComparison(objid, ws, service=self.service)
objid, ws = _translate_obj_identity(params['genome_workspace'],
params['genome_name'])
genome = Genome(objid, ws, service=self.service)
probanno = None
if 'rxn_probs_name' in params and 'rxn_probs_workspace' in params and \
params['rxn_probs_name'] is not None and len(params['rxn_probs_name']) > 0:
objid, ws = _translate_obj_identity(params['rxn_probs_workspace'],
params['rxn_probs_name'])
probanno = ReactionProbabilities(objid, ws, service=self.service)
morph = Morph(service=self.service,
src_model=model,
media=media,
probanno=probanno,
protcomp=protcomp,
genome=genome,
ws_id=params['workspace'])
if 'fill_src' in params and params['fill_src']:
morph.fill_src_to_media()
morph.translate_features()
morph.reconstruct_genome()
morph.label_reactions()
morph.build_supermodel()
if 'translate_media' in params and params['translate_media']:
if 'target_media_name' in params and 'target_media_workspace' in params:
objid, ws = self._translate_obj_identity(
params['target_media_workspace'],
params['target_media_name'])
new_media = Media(objid, ws, service=self.service)
else:
new_media = morph.media
morph.translate_media(new_media)
output_name = params[
'output_name'] if 'output_name' in params else 'MorphedModel'
if 'num_reactions_to_process' in params:
morph.process_reactions(num_reactions=int(
params['num_reactions_to_process']),
name=output_name)
else:
morph.process_reactions(name=output_name)
reportObj = {'objects_created': [], 'text_message': "MIGHTY"}
#save report
provenance = [{}]
if 'provenance' in ctx:
provenance = ctx['provenance']
# add additional info to provenance here, in this case the input data object reference
provenance[0]['input_ws_objects'] = [
params['workspace'] + '/' + params['fbamodel_name']
]
try:
report_info_list = self.service.ws_client.save_objects({
'workspace':
params['workspace'],
'objects': [{
'type':
'KBaseReport.Report',
'data':
reportObj,
'name':
'CallingFBA_report' + str(hex(uuid.getnode())),
'meta': {},
'hidden':
1, # important! make sure the report is hidden
'provenance':
provenance
}]
})
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
orig_error = ''.join(' ' + line for line in lines)
raise ValueError('Error saving Report object to workspace:\n' +
orig_error)
report_info = report_info_list[0]
print('Ready to return')
returnVal = {
'report_name':
'FS_report',
'report_ref':
str(report_info[6]) + '/' + str(report_info[0]) + '/' +
str(report_info[4])
}
#END morph_model
# At some point might do deeper type checking...
if not isinstance(returnVal, dict):
raise ValueError('Method morph_model return value ' +
'returnVal is not type dict as required.')
# return the results
return [returnVal]
def __init__(self):
# self.parse_text() # ai.ja.txtをparseする
self.morph = Morph()
def init_morph():
global morph
if not morph: morph = Morph()
(u"man", u"manmandə"),
# (u"kwaj",u"kwajkwajdə"),
# (u"çin",u"çinçində"),
(u"le", u"leledə")
]
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Learn pig Latin and Chinese")
parser.add_argument(
'task',
choices=["Chinese", "Latin", "Latin1", "Latin2", "Latin3"],
default="Latin")
parser.add_argument("-d", "--depth", default=1, type=int)
arguments = parser.parse_args()
examples = data[arguments.task]
depth = arguments.depth
leaveSketchOutput()
solution = SupervisedProblem(
[(Morph(tokenize(x)), Constant(0), Morph(tokenize(y)))
for x, y in examples],
syllables=True).solve(depth)
if solution == None:
print "No solution."
else:
for r in solution:
print r
def affix():
if useMorphology: return Morph.sample()
else: return Morph([]).makeConstant(self.bank)
def toMorph(z):
if isinstance(z, Morph): return z
elif isinstance(z, (unicode, str)): return Morph(tokenize(z))
else: assert False
def __init__(self, morph=None):
self.morph = morph or Morph()
def parseAffix(output, morph):
if useMorphology: return Morph.parse(self.bank, output, morph)
else: return Morph([])
def parse_morph(self, line):
surface, result = line.split()
results = result.split(',')
morph = Morph(surface, results[6], results[0], results[1])
return morph
def restrict(self, newData):
"""Creates a new version of this object which is identical but has different training data"""
restriction = copy.copy(self)
restriction.data = [ tuple( None if i == None else (i if isinstance(i,Morph) else Morph(tokenize(i)))
for i in Lex)
for Lex in newData ]
return restriction
parser.add_option(
'-c',
'--wordclass',
dest='wordclass',
default='',
help=
"limit tests to specific word class ('n' for noun, 's' for verb etc.)")
parser.add_option(
'-l',
'--loose',
dest='loose',
action="store_true",
help="allow a more 'loose' comparison, omitting certain parts of tags")
options, args = parser.parse_args()
m = Morph()
total = 0.0
total_full = 0.0
total_tags = 0.0
total_tags_full = 0
tags_missed = 0
missing = 0
all_missing = 0
no_candidates = 0
surplus = 0
wrong_pick = 0
tags_ignored = 0
words_ignored = 0
start_time = time.time()