def comparison_detail ():
"""Output comparison of 2 witnesses, chapter detail.
Outputs a detail of the differences between 2 manuscripts in one chapter.
"""
with current_app.config.dba.engine.begin () as conn:
ms1 = Manuscript (conn, request.args.get ('ms1') or 'A')
ms2 = Manuscript (conn, request.args.get ('ms2') or 'A')
range_ = request.args.get ('range') or 'All'
res = execute (conn, """
SELECT p.pass_id, p.begadr, p.endadr, v1.labez_clique, v1.lesart,
v2.labez_clique, v2.lesart,
is_p_older (p.pass_id, v1.labez, v1.clique, v2.labez, v2.clique) AS older,
is_p_older (p.pass_id, v2.labez, v2.clique, v1.labez, v1.clique) AS newer,
is_p_unclear (p.pass_id, v1.labez, v1.clique) OR
is_p_unclear (p.pass_id, v2.labez, v2.clique) AS unclear
FROM (SELECT * FROM ranges WHERE range = :range_) r
JOIN passages p ON (r.passage @> p.passage )
JOIN apparatus_cliques_view v1 USING (pass_id)
JOIN apparatus_cliques_view v2 USING (pass_id)
WHERE v1.ms_id = :ms1 AND v2.ms_id = :ms2
AND v1.labez != v2.labez AND v1.labez !~ '^z' AND v2.labez !~ '^z'
AND v1.cbgm AND v2.cbgm
ORDER BY p.pass_id
""", dict (parameters, ms1 = ms1.ms_id, ms2 = ms2.ms_id, range_ = range_))
return list (map (_ComparisonDetailRowCalcFields._make, res))
def optimal_substemma_json():
"""Normalize parameters only and add some general info.
"""
if current_app.config.val is None:
current_app.config.val = init(current_app.config.dba)
val = current_app.config.val
with current_app.config.dba.engine.begin() as conn:
# the manuscript to explain
ms = Manuscript(conn, request.args.get('ms'))
# get the selected set of ancestors and build all combinations of that set
selected = [
Manuscript(conn, anc_id)
for anc_id in (request.args.get('selection') or '').split()
]
response = {
'ms': ms.to_json(),
'mss': [s.to_json() for s in selected],
}
n_defined = np.count_nonzero(val.def_matrix[ms.ms_id - 1])
response['ms']['open'] = n_defined
return make_json_response(response)
def comparison_summary ():
"""Output comparison of 2 witnesses, chapter summary.
Outputs a summary of the differences between 2 manuscripts, one summary row
per chapters.
"""
with current_app.config.dba.engine.begin () as conn:
ms1 = Manuscript (conn, request.args.get ('ms1') or 'A')
ms2 = Manuscript (conn, request.args.get ('ms2') or 'A')
res = execute (conn, """
(WITH ranks AS (
SELECT ms_id1, ms_id2, rg_id, rank () OVER (PARTITION BY rg_id ORDER BY affinity DESC) AS rank, affinity
FROM affinity aff
WHERE ms_id1 = :ms_id1
AND {prefix}newer > {prefix}older
ORDER BY affinity DESC
)
SELECT a.rg_id, a.range, a.common, a.equal,
a.older, a.newer, a.unclear, a.affinity, r.rank, ms1_length, ms2_length
FROM {view} a
JOIN ranks r USING (rg_id, ms_id1, ms_id2)
WHERE a.ms_id1 = :ms_id1 AND a.ms_id2 = :ms_id2
)
UNION
(WITH ranks2 AS (
SELECT ms_id1, ms_id2, rg_id, rank () OVER (PARTITION BY rg_id ORDER BY affinity DESC) AS rank, affinity
FROM affinity aff
WHERE ms_id2 = :ms_id2
AND {prefix}newer < {prefix}older
ORDER BY affinity DESC
)
SELECT a.rg_id, a.range, a.common, a.equal,
a.older, a.newer, a.unclear, a.affinity, r.rank, ms1_length, ms2_length
FROM {view} a
JOIN ranks2 r USING (rg_id, ms_id1, ms_id2)
WHERE a.ms_id1 = :ms_id1 AND a.ms_id2 = :ms_id2
)
UNION
SELECT a.rg_id, a.range, a.common, a.equal,
a.older, a.newer, a.unclear, a.affinity, NULL, ms1_length, ms2_length
FROM {view} a
WHERE a.ms_id1 = :ms_id1 AND a.ms_id2 = :ms_id2 AND a.newer = a.older
ORDER BY rg_id
""", dict (parameters, ms_id1 = ms1.ms_id, ms_id2 = ms2.ms_id,
view = 'affinity_p_view', prefix = 'p_'))
return list (map (_ComparisonRowCalcFields._make, res))
def manuscript_full_json (passage_or_id, hs_hsnr_id):
"""Endpoint. Serve information about a manuscript.
:param string hs_hsnr_id: The hs, hsnr or id of the manuscript.
"""
auth ()
hs_hsnr_id = request.args.get ('ms_id') or hs_hsnr_id
chapter = request.args.get ('range') or 'All'
with current_app.config.dba.engine.begin () as conn:
passage = Passage (conn, passage_or_id)
ms = Manuscript (conn, hs_hsnr_id)
rg_id = passage.range_id (chapter)
json = ms.to_json ()
json['length'] = ms.get_length (passage, chapter)
# Get the attestation(s) of the manuscript (may be uncertain eg. a/b/c)
res = execute (conn, """
SELECT labez, clique, labez_clique
FROM apparatus_view_agg
WHERE ms_id = :ms_id AND pass_id = :pass_id
""", dict (parameters, ms_id = ms.ms_id, pass_id = passage.pass_id))
json['labez'], json['clique'], json['labez_clique'] = res.fetchone ()
# Get the affinity of the manuscript to all manuscripts
res = execute (conn, """
SELECT avg (a.affinity) as aa,
percentile_cont(0.5) WITHIN GROUP (ORDER BY a.affinity) as ma
FROM affinity a
WHERE a.ms_id1 = :ms_id1 AND a.rg_id = :rg_id
""", dict (parameters, ms_id1 = ms.ms_id, rg_id = rg_id))
json['aa'], json['ma'] = res.fetchone ()
# Get the affinity of the manuscript to MT
#
# For a description of mt and mtp see the comment in
# ActsMsListValPh3.pl and
# http://intf.uni-muenster.de/cbgm/actsPh3/guide_en.html#Ancestors
json['mt'], json['mtp'] = 0.0, 0.0
res = execute (conn, """
SELECT a.affinity as mt, a.equal::float / c.length as mtp
FROM affinity a
JOIN ms_ranges c
ON (a.ms_id1, a.rg_id) = (c.ms_id, c.rg_id)
WHERE a.ms_id1 = :ms_id1 AND a.ms_id2 = 2 AND a.rg_id = :rg_id
""", dict (parameters, ms_id1 = ms.ms_id, rg_id = rg_id))
if res.rowcount > 0:
json['mt'], json['mtp'] = res.fetchone ()
return make_json_response (json)
def manuscript_json (hs_hsnr_id):
"""Endpoint. Serve information about a manuscript.
:param string hs_hsnr_id: The hs, hsnr or id of the manuscript.
"""
auth ()
hs_hsnr_id = request.args.get ('ms_id') or hs_hsnr_id
with current_app.config.dba.engine.begin () as conn:
ms = Manuscript (conn, hs_hsnr_id)
return make_json_response (ms.to_json ())
def manuscript_json(hs_hsnr_id):
"""Endpoint. Serve information about a manuscript.
:param string hs_hsnr_id: The hs, hsnr or id of the manuscript.
"""
auth()
hs_hsnr_id = request.args.get('ms_id') or hs_hsnr_id
with current_app.config.dba.engine.begin() as conn:
ms = Manuscript(conn, hs_hsnr_id)
return make_json_response(ms.to_json())
def optimal_substemma_detail_csv():
"""Report details about one combination of ancestors.
"""
if current_app.config.val is None:
current_app.config.val = init(current_app.config.dba)
val = current_app.config.val
with current_app.config.dba.engine.begin() as conn:
# the manuscript to explain
ms = Manuscript(conn, request.args.get('ms'))
# get the selected set of ancestors
selected = [
Manuscript(conn, anc_id)
for anc_id in (request.args.get('selection') or '').split()
]
combinations = [Combination(selected, 0)]
explain_matrix = build_explain_matrix(conn, val, ms.ms_id)
_optimal_substemma(ms.ms_id,
explain_matrix,
combinations,
mode='detail')
res = execute(
conn, """
SELECT 'unknown' as type, p.pass_id, p.begadr, p.endadr, v.labez_clique, v.lesart
FROM passages p
JOIN apparatus_cliques_view v USING (pass_id)
WHERE v.ms_id = :ms_id AND pass_id IN :unknown_pass_ids
UNION
SELECT 'open' as type, p.pass_id, p.begadr, p.endadr, v.labez_clique, v.lesart
FROM passages p
JOIN apparatus_cliques_view v USING (pass_id)
WHERE v.ms_id = :ms_id AND pass_id IN :open_pass_ids
""",
dict(ms_id=ms.ms_id,
unknown_pass_ids=combinations[0].unknown_indices or (-1, ),
open_pass_ids=combinations[0].open_indices or (-1, )))
return csvify(
_OptimalSubstemmaDetailRowCalcFields._fields,
list(map(_OptimalSubstemmaDetailRowCalcFields._make, res)))
def optimal_substemma_json ():
"""Normalize parameters and add some general info.
"""
if current_app.config.val is None:
current_app.config.val = init (current_app.config.dba)
val = current_app.config.val
with current_app.config.dba.engine.begin () as conn:
# the manuscript to explain
ms = Manuscript (conn, request.args.get ('ms'))
# get the selected set of ancestors and build all combinations of that set
selected = [ Manuscript (conn, anc_id)
for anc_id in (request.args.get ('selection') or '').split () ]
response = {
'ms' : ms.to_json (),
'mss' : [s.to_json () for s in selected],
}
n_defined = np.count_nonzero (val.def_matrix[ms.ms_id - 1])
response['ms']['open'] = n_defined
return make_json_response (response)
def optimal_substemma_csv():
"""Do an exhaustive search for the combination among a given set of ancestors
that best explains a given manuscript.
"""
if current_app.config.val is None:
current_app.config.val = init(current_app.config.dba)
val = current_app.config.val
with current_app.config.dba.engine.begin() as conn:
# the manuscript to explain
ms = Manuscript(conn, request.args.get('ms'))
# get the selected set of ancestors and build all combinations of that set
selected = [
Manuscript(conn, anc_id)
for anc_id in (request.args.get('selection') or '').split()
]
combinations = []
i = 0
for l in range(len(selected)):
for c in itertools.combinations(selected, l + 1):
combinations.append(Combination(c, i))
i += 1
explain_matrix = build_explain_matrix(conn, val, ms.ms_id)
_optimal_substemma(ms.ms_id,
explain_matrix,
combinations,
mode='search')
res = [c.to_csv() for c in combinations]
return csvify(_OptimalSubstemmaRow._fields,
list(map(_OptimalSubstemmaRow._make, res)))
def set_cover_json (hs_hsnr_id):
""" Approximate the minimum set cover for a manuscript.
See: https://en.wikipedia.org/wiki/Set_cover_problem
"""
response = {}
with current_app.config.dba.engine.begin () as conn:
if current_app.config.val is None:
current_app.config.val = init (current_app.config.dba)
val = current_app.config.val
cover = []
ms = Manuscript (conn, hs_hsnr_id)
response['ms'] = ms.to_json ()
ms_id = ms.ms_id - 1 # numpy indices start at 0
# allow user to pre-select a set of manuscripts
pre_selected = [ Manuscript (conn, anc_id)
for anc_id in (request.args.get ('pre_select') or '').split () ]
response['mss'] = [s.to_json () for s in pre_selected]
np.set_printoptions (edgeitems = 8, linewidth = 100)
b_common = np.logical_and (val.def_matrix, val.def_matrix[ms_id])
b_common[ms_id] = False # don't find original ms.
b_common[0] = False # don't find A
b_common[1] = False # don't find MT
# eliminate descendants from the matrix
ancestors = get_ancestors (conn, current_app.config.set_cover_rg_id, ms.ms_id)
for i in range (0, val.n_mss):
if (i + 1) not in ancestors:
b_common[i] = 0
n_defined = np.count_nonzero (val.def_matrix[ms_id])
response['ms']['open'] = n_defined
explain_matrix = build_explain_matrix (conn, val, ms.ms_id)
explain_equal_matrix = val.mask_matrix[ms_id]
# The mss x passages boolean matrix that is TRUE whenever the inspected ms.
# agrees with the potential source ms.
b_equal = np.bitwise_and (val.mask_matrix, explain_equal_matrix) > 0
b_equal = np.logical_and (b_equal, b_common)
# The mss x passages boolean matrix that is TRUE whenever the inspected ms.
# agrees with the potential source ms. or is posterior to it.
b_post = np.bitwise_and (val.mask_matrix, explain_matrix) > 0
b_post = np.logical_and (b_post, b_common)
n_explained = 0
for n in range (0, MAX_COVER_SIZE):
if n < len (pre_selected):
# use manuscript pre-selected by user
ms_id_most_similar = pre_selected[n].ms_id - 1
else:
# find manuscript that explains the most passages by agreement
ms_id_most_similar = int (np.argmax (np.sum (b_equal, axis = 1)))
b_explained = np.copy (b_post[ms_id_most_similar])
b_explained_equal = np.copy (b_equal[ms_id_most_similar])
ms_most_similar = Manuscript (conn, 'id' + str (ms_id_most_similar + 1))
d = ms_most_similar.to_json ()
# exit if no passages could be explained
n_explains = int (np.count_nonzero (b_explained))
n_equal = int (np.count_nonzero (b_explained_equal))
if n_explains == 0:
break
n_explained += n_explains
# remove "explained" readings, so they will not be matched again
b_post[:,b_explained] = False
b_equal[:,b_explained] = False
explain_matrix[b_explained] = 0
# explain_equal_matrix[b_explained] = 0
n_unknown = np.count_nonzero (np.bitwise_and (explain_matrix, 0x1))
d['explains'] = n_explains
d['equal'] = n_equal
d['post'] = n_explains - n_equal
d['explained'] = n_explained
d['unknown'] = n_unknown
d['open'] = n_defined - n_explained - n_unknown
d['n'] = n + 1
cover.append (d)
# output list
response['cover'] = cover
return make_json_response (response)
def relatives_csv(passage_or_id, hs_hsnr_id):
"""Output a table of the nearest relatives of a manuscript.
Output a table of the nearest relatives/ancestors/descendants of a
manuscript and what they attest.
"""
auth()
type_ = request.args.get('type') or 'rel'
limit = int(request.args.get('limit') or 0)
labez = request.args.get('labez') or 'all'
mode = request.args.get('mode') or 'sim'
include = request.args.getlist('include[]') or []
fragments = request.args.getlist('fragments[]') or []
view = 'affinity_view' if mode == 'rec' else 'affinity_p_view'
where = ''
if type_ == 'anc':
where = ' AND older < newer'
if type_ == 'des':
where = ' AND older >= newer'
if labez == 'all':
where += " AND labez !~ '^z'"
elif labez == 'all+lac':
pass
else:
where += " AND labez = '%s'" % labez
if 'fragments' in fragments:
frag_where = ''
else:
frag_where = 'AND aff.common > aff.ms1_length / 2'
limit = '' if limit == 0 else ' LIMIT %d' % limit
with current_app.config.dba.engine.begin() as conn:
passage = Passage(conn, passage_or_id)
ms = Manuscript(conn, hs_hsnr_id)
rg_id = passage.request_rg_id(request)
exclude = get_excluded_ms_ids(conn, include)
# Get the X most similar manuscripts and their attestations
res = execute(
conn, """
/* get the LIMIT closest ancestors for this node */
WITH ranks AS (
SELECT ms_id1, ms_id2,
rank () OVER (ORDER BY affinity DESC, common, older, newer DESC, ms_id2) AS rank,
affinity
FROM {view} aff
WHERE ms_id1 = :ms_id1 AND aff.rg_id = :rg_id AND ms_id2 NOT IN :exclude
AND newer > older {frag_where}
ORDER BY affinity DESC
)
SELECT r.rank,
aff.ms_id2 as ms_id,
ms.hs,
ms.hsnr,
aff.ms2_length,
aff.common,
aff.equal,
aff.older,
aff.newer,
aff.unclear,
aff.common - aff.equal - aff.older - aff.newer - aff.unclear as norel,
CASE WHEN aff.newer < aff.older THEN ''
WHEN aff.newer = aff.older THEN '-'
ELSE '>'
END as direction,
aff.affinity,
a.labez,
a.certainty
FROM
{view} aff
JOIN apparatus_view_agg a
ON aff.ms_id2 = a.ms_id
JOIN manuscripts ms
ON aff.ms_id2 = ms.ms_id
LEFT JOIN ranks r
ON r.ms_id2 = aff.ms_id2
WHERE aff.ms_id2 NOT IN :exclude AND aff.ms_id1 = :ms_id1
AND aff.rg_id = :rg_id AND aff.common > 0
AND a.pass_id = :pass_id {where} {frag_where}
ORDER BY affinity DESC, r.rank, newer DESC, older DESC, hsnr
{limit}
""",
dict(parameters,
where=where,
frag_where=frag_where,
ms_id1=ms.ms_id,
hsnr=ms.hsnr,
pass_id=passage.pass_id,
rg_id=rg_id,
limit=limit,
view=view,
exclude=exclude))
Relatives = collections.namedtuple(
'Relatives',
'rank ms_id hs hsnr length common equal older newer unclear norel direction affinity labez certainty'
)
return csvify(Relatives._fields, list(map(Relatives._make, res)))
def manuscript_full_json(passage_or_id, hs_hsnr_id):
"""Endpoint. Serve information about a manuscript.
:param string hs_hsnr_id: The hs, hsnr or id of the manuscript.
"""
auth()
hs_hsnr_id = request.args.get('ms_id') or hs_hsnr_id
with current_app.config.dba.engine.begin() as conn:
passage = Passage(conn, passage_or_id)
ms = Manuscript(conn, hs_hsnr_id)
rg_id = passage.request_rg_id(request)
if ms.ms_id is None:
return cache(
make_json_response(None, 400,
'Bad request: No such manuscript.'))
json = ms.to_json()
json['length'] = ms.get_length(rg_id)
# Get the attestation(s) of the manuscript (may be uncertain eg. a/b/c)
res = execute(
conn, """
SELECT labez, clique, labez_clique, certainty
FROM apparatus_view_agg
WHERE ms_id = :ms_id AND pass_id = :pass_id
""", dict(parameters, ms_id=ms.ms_id, pass_id=passage.pass_id))
row = res.fetchone()
if row is not None:
json['labez'], json['clique'], json['labez_clique'], json[
'certainty'] = row
# Get the affinity of the manuscript to all manuscripts
res = execute(
conn, """
SELECT avg (a.affinity) as aa,
percentile_cont(0.5) WITHIN GROUP (ORDER BY a.affinity) as ma
FROM affinity a
WHERE a.ms_id1 = :ms_id1 AND a.rg_id = :rg_id
""", dict(parameters, ms_id1=ms.ms_id, rg_id=rg_id))
json['aa'], json['ma'] = 0.0, 0.0
row = res.fetchone()
if row is not None:
json['aa'], json['ma'] = row
# Get the affinity of the manuscript to MT
#
# For a description of mt and mtp see the comment in
# ActsMsListValPh3.pl and
# http://intf.uni-muenster.de/cbgm/actsPh3/guide_en.html#Ancestors
res = execute(
conn, """
SELECT a.affinity as mt, a.equal::float / c.length as mtp
FROM affinity a
JOIN ms_ranges c
ON (a.ms_id1, a.rg_id) = (c.ms_id, c.rg_id)
WHERE a.ms_id1 = :ms_id1 AND a.ms_id2 = 2 AND a.rg_id = :rg_id
""", dict(parameters, ms_id1=ms.ms_id, rg_id=rg_id))
json['mt'], json['mtp'] = 0.0, 0.0
row = res.fetchone()
if row is not None:
json['mt'], json['mtp'] = row
return cache(make_json_response(json))
def set_cover_json(hs_hsnr_id):
""" Approximate the minimum set cover for a manuscript.
See: https://en.wikipedia.org/wiki/Set_cover_problem
"""
include = request.args.getlist('include[]') or []
pre_select = (request.args.get('pre_select') or '').split()
response = {}
with current_app.config.dba.engine.begin() as conn:
if current_app.config.val is None:
current_app.config.val = init(current_app.config.dba)
val = current_app.config.val
cover = []
ms = Manuscript(conn, hs_hsnr_id)
response['ms'] = ms.to_json()
ms_id = ms.ms_id - 1 # numpy indices start at 0
# allow user to pre-select a set of manuscripts
pre_selected = [Manuscript(conn, anc_id) for anc_id in pre_select]
response['mss'] = [s.to_json() for s in pre_selected]
np.set_printoptions(edgeitems=8, linewidth=100)
# The mss x passages boolean matrix that is TRUE whenever the inspected
# ms. and the source ms. are both defined.
b_common = np.logical_and(val.def_matrix, val.def_matrix[ms_id])
# Remove mss. we don't want to compare
b_common[ms_id] = False # don't find original ms.
if 'A' not in include and 'A' not in pre_select:
b_common[0] = False
if 'MT' not in include and 'MT' not in pre_select:
b_common[1] = False
# also eliminate all descendants
ancestors = get_ancestors(conn, current_app.config.rg_id_all, ms.ms_id)
for i in range(0, val.n_mss):
if (i + 1) not in ancestors:
b_common[i] = False
n_defined = np.count_nonzero(val.def_matrix[ms_id])
response['ms']['open'] = n_defined
# mask_matrix ist the mss x passages matrix containing the bitmask of
# all readings
explain_matrix = build_explain_matrix(conn, val, ms.ms_id)
explain_equal_matrix = val.mask_matrix[ms_id]
# The mss x passages boolean matrix that is TRUE whenever the inspected ms.
# agrees with the potential source ms.
b_equal = np.bitwise_and(val.mask_matrix, explain_equal_matrix) > 0
b_equal = np.logical_and(b_equal, b_common)
# The mss x passages boolean matrix that is TRUE whenever the inspected ms.
# agrees with the potential source ms. or is posterior to it.
b_post = np.bitwise_and(val.mask_matrix, explain_matrix) > 0
b_post = np.logical_and(b_post, b_common)
# The 1 x passages boolean matrix that is TRUE whenever the passage is
# still unexplained.
b_open = np.copy(val.def_matrix[ms_id])
# The 1 x passages boolean matrix that is TRUE whenever the source of
# the reading in the inspected ms. is unknown
b_unknown = np.bitwise_and(explain_matrix, 0x1)
b_unknown = np.logical_and(b_unknown, b_open)
n_explained = 0
for n in range(0, MAX_COVER_SIZE):
if n < len(pre_selected):
# use manuscript pre-selected by user
ms_id_most_similar = pre_selected[n].ms_id - 1
else:
# find manuscript that explains the most passages by agreement
ms_id_most_similar = int(np.argmax(np.sum(b_equal, axis=1)))
ms_most_similar = Manuscript(conn,
'id' + str(ms_id_most_similar + 1))
d = ms_most_similar.to_json()
b_explained = np.copy(b_post[ms_id_most_similar])
n_explains = int(np.count_nonzero(b_explained))
# exit if no passages could be explained
if n_explains == 0:
break
n_equal = int(np.count_nonzero(b_equal[ms_id_most_similar]))
# remove "explained" readings, so they will not be matched again
b_post[:, b_explained] = False
b_equal[:, b_explained] = False
b_unknown[b_explained] = False
b_open[b_explained] = False
n_explained += n_explains
n_unknown = int(np.count_nonzero(b_unknown))
n_open = int(np.count_nonzero(b_open))
d['explains'] = n_explains
d['explained'] = n_explained
d['equal'] = n_equal
d['post'] = n_explains - n_equal
d['unknown'] = n_unknown
d['open'] = n_open - n_unknown
d['n'] = n + 1
cover.append(d)
# output list
response['cover'] = cover
return make_json_response(response)