def instances_for_model(model, fields=None, limit=20, offset=0):
if not fields:
fields = fields_for_model(model)
attribute_from_field = par(field_to_attribute, model)
keywordify = comp(K, attribute_from_field)
entity_from_field = comp(get_entity, keywordify)
rules = map(entity_from_field, fields)
query = [K("find"), S("?touched"), K("in"), S("$"), S("%"),
K("where"), S("(get-entity ?eid)"),
[S("(datomic.api/entity $ ?eid)"), S("?e")],
[S("(datomic.api/touch ?e)"), S("?touched")]]
return query_fn(Config().berossus_url, query,
params=[rules], limit=limit, offset=offset)
def summ(step, wtr = tf.summary.FileWriter(pform(P.log, C.trial))
, summary = tf.summary.merge(
( tf.summary.scalar('step_errt', model.errt)
, tf.summary.scalar('step_loss', model.loss)))):
errt, loss = map(comp(np.mean, np.concatenate), zip(*chain(*(
batch_run(sess, m, (m.errt_samp, m.loss_samp), s, t, batch= C.batch_valid)
for m, (s, t) in zip(valid, data_valid)))))
wtr.add_summary(sess.run(summary, {model.errt: errt, model.loss: loss}), step)
wtr.flush()
def load_batch(names, load=comp(np.load, "trial/data/grams/{}.npy".format)):
names = names.astype(np.str)
x = vpack(map(load, names), complex('(nan+nanj)'), 1, 1)
# x = vpack(map(comp(load, path), names), complex('(nan+nanj)'), 1, 1)
x[:, 0] = 0j
x = c2r(x)
_, t, d = x.shape
assert t <= len_cap
assert d == dim_tgt
return x
def instance(id):
page_desc = "Viewing instance id: " + str(id)
extra_bc = request.args.get("breadcrumb")
breadcrumbs = [("Home", url_for("index"))]
if extra_bc:
breadcrumbs.append(extra_bc)
this = models.values_for_instance(id)[K("result")][0][0]
fields = this.keys()
headers = map(comp(Item, str), fields)
data = [map(lambda x: Item(this[x]), this)]
return render_template('table.html', **locals())
def reapplySequence(self):
"""
"""
compSS = self.strandSet().complementStrandSet()
# the strand sequence will need to be regenerated from scratch
# as there are no guarantees about the entirety of the strand moving
# i.e. both endpoints thanks to multiple selections so just redo the
# whole thing
self._sequence = None
for compStrand in compSS._findOverlappingRanges(self):
compSeq = compStrand.sequence()
usedSeq = util.comp(compSeq) if compSeq else None
usedSeq = self.setComplementSequence(
usedSeq, compStrand)
def reapplySequence(self):
"""
"""
compSS = self.strandSet().complementStrandSet()
# the strand sequence will need to be regenerated from scratch
# as there are no guarantees about the entirety of the strand moving
# i.e. both endpoints thanks to multiple selections so just redo the
# whole thing
self._sequence = None
for compStrand in compSS._findOverlappingRanges(self):
compSeq = compStrand.sequence()
usedSeq = util.comp(compSeq) if compSeq else None
usedSeq = self.setComplementSequence(
usedSeq, compStrand)
def summ(step, model=model_valid):
wtr.add_summary(
sess.run(
summary,
dict(
zip((model.errt, model.loss_gen, model.loss_kld),
map(
comp(np.mean, np.concatenate),
zip(*(sess.run((model.errt_samp, model.loss_gen_samp,
model.loss_kld_samp), {
model.src: valid[i:j],
model.tgt: valid[i:j]
})
for i, j in partition(
len(valid), T.batch_valid, discard=False)
)))))), step)
wtr.flush()
def undo(self):
olg = self._oligo
oS = ''.join(self._oldSequence) if self._oldSequence else None
oligoList = [olg]
for strand in olg.strand5p().generator3pStrand():
usedSeq, oS = strand.setSequence(oS)
# get the compliment ahead of time
usedSeq = util.comp(usedSeq) if usedSeq else None
compSS = strand.strandSet().complementStrandSet()
for compStrand in compSS._findOverlappingRanges(strand):
subUsedSeq = compStrand.setComplementSequence(usedSeq, strand)
oligoList.append(compStrand.oligo())
# end for
# for
for oligo in oligoList:
oligo.oligoSequenceAddedSignal.emit(oligo)
def redo(self):
olg = self._oligo
nS = ''.join(self._newSequence) if self._newSequence else None
nS_original = self._newSequence
oligoList = [olg]
for strand in olg.strand5p().generator3pStrand():
usedSeq, nS = strand.setSequence(nS)
# get the compliment ahead of time
usedSeq = util.comp(usedSeq) if usedSeq else None
compSS = strand.strandSet().complementStrandSet()
for compStrand in compSS._findOverlappingRanges(strand):
subUsedSeq = compStrand.setComplementSequence(usedSeq, strand)
oligoList.append(compStrand.oligo())
# end for
# as long as the new Applied Sequence is not None
if nS == None and nS_original:
break
# end for
for oligo in oligoList:
oligo.oligoSequenceAddedSignal.emit(oligo)
def undo(self):
olg = self._oligo
oS = ''.join(self._oldSequence) if self._oldSequence else None
oligoList = [olg]
for strand in olg.strand5p().generator3pStrand():
usedSeq, oS = strand.setSequence(oS)
# get the compliment ahead of time
usedSeq = util.comp(usedSeq) if usedSeq else None
compSS = strand.strandSet().complementStrandSet()
for compStrand in compSS._findOverlappingRanges(strand):
subUsedSeq = compStrand.setComplementSequence(
usedSeq, strand)
oligoList.append(compStrand.oligo())
# end for
# for
for oligo in oligoList:
oligo.oligoSequenceAddedSignal.emit(oligo)
def redo(self):
olg = self._oligo
nS = ''.join(self._newSequence) if self._newSequence else None
nS_original = self._newSequence
oligoList = [olg]
for strand in olg.strand5p().generator3pStrand():
usedSeq, nS = strand.setSequence(nS)
# get the compliment ahead of time
usedSeq = util.comp(usedSeq) if usedSeq else None
compSS = strand.strandSet().complementStrandSet()
for compStrand in compSS._findOverlappingRanges(strand):
subUsedSeq = compStrand.setComplementSequence(
usedSeq, strand)
oligoList.append(compStrand.oligo())
# end for
# as long as the new Applied Sequence is not None
if nS == None and nS_original:
break
# end for
for oligo in oligoList:
oligo.oligoSequenceAddedSignal.emit(oligo)
def field_instances_for_model(model):
attributes = map(par(field_to_attribute, model), fields_for_model(model))
return map(comp(grab_single, instance_for_ident), attributes)
def grab_single(result):
return comp(ffirst, par(nth, K("result")))(result)
valid_en, train_en = np.load(pform(P.data, "valid_en.npy")), np.load(pform(P.data, "train_en.npy"))
# valid_nl, train_nl = np.load(pform(P.data, "valid_nl.npy")), np.load(pform(P.data, "train_nl.npy"))
valid_de, train_de = np.load(pform(P.data, "valid_de.npy")), np.load(pform(P.data, "train_de.npy"))
# valid_da, train_da = np.load(pform(P.data, "valid_da.npy")), np.load(pform(P.data, "train_da.npy"))
valid_sv, train_sv = np.load(pform(P.data, "valid_sv.npy")), np.load(pform(P.data, "train_sv.npy"))
data_index = 0, 2, 4
data_valid = valid_en, valid_de, valid_sv
data_train = train_en, train_de, train_sv
def batch(arrs, size= C.batch_train, seed= C.seed):
size //= len(arrs) * (len(arrs) - 1)
for i in batch_sample(len(arrs[0]), size, seed):
yield tuple(arr[i] for arr in arrs)
perm = comp(tuple, partial(permutations, r= 2))
data_index = perm(data_index)
data_valid = perm(data_valid)
data_train = perm(pipe(partial(batch, data_train), (tf.int32,)*len(data_train), prefetch= 16))
###############
# build model #
###############
model = Model.new(**select(C, *Model._new))
valid = tuple(model.data(i, j).valid() for i, j in data_index)
train = tuple(model.data(i, j, s, t).train(**T) for (i, j), (s, t) in zip(data_index, data_train))
model.lr = train[0].lr
model.step = train[0].step
model.errt = train[0].errt
def bleu(gold, pred):
"""-> float in [0, 1]; gold, pred : seq (sent : seq (word : str))"""
from nltk.translate.bleu_score import corpus_bleu
return corpus_bleu([[g] for g in gold], pred)
def parse_args():
import argparse
parser = argparse.ArgumentParser(description="corpus-level BLEU score.")
parser.add_argument('gold', help="file for the gold-standard sentences")
parser.add_argument('pred',
help="files for the predicted sentences",
nargs='+')
parser.add_argument('--ignore-case',
action='store_true',
help="case insensitive")
return parser.parse_args()
if '__main__' == __name__:
args = parse_args()
from util import comp, partial
from util_io import load
proc = str.split
if args.ignore_case: proc = comp(proc, str.lower)
load_corpus = comp(list, partial(map, proc), load)
gold = load_corpus(args.gold)
for pred in args.pred:
print(pred, "{:.4f}".format(bleu(gold, load_corpus(pred))))
#!/usr/bin/env python3
from util import comp, partial, PointedIndex
from util_io import path, load_meta, load
from util_np import np, vpack
names, texts = load_meta()
chars = {char for text in texts for char in text}
chars.remove("\n")
chars.remove(" ")
index = PointedIndex(" \n" + "".join(sorted(chars)))
texts = vpack(
map(comp(partial(np.fromiter, dtype=np.uint8), partial(map, index)),
texts), index("\n"))
np.save("trial/data/index", index.vec)
np.save("trial/data/texts", texts)
np.save("trial/data/names", names)
for name in names:
np.save("trial/data/grams/" + name, load(path(name)))
