def test_07_coll_2d_check_values(self):
"""Cell-by-cell value comparison for 2-D arrays
This test is complicated by the fact that with different chunk
intervals, redimension() does not produce synthetic dimension
siblings in any particular order. So we must process the
filtered list of differing cells, and only complain if the
*set* of values along the synthetic dimension at [x,y,*]
differs for the two arrays. For example, if the two arrays
held
{x,y,synth} v {x,y,synth} v
{2,7,0} 20 {2,7,0} 20
{2,7,1} 73 {2,7,1} 99
{2,7,2} 99 {2,7,2} 73
that is perfectly fine.
"""
tbl = make_table('CellDiffs', """
filter(join(coll_2d,coll_2d_ac), coll_2d.v <> coll_2d_ac.v)
""")
v_xy_sets = defaultdict(set)
v2_xy_sets = defaultdict(set)
for celldiff in tbl:
key = (int(celldiff.x), int(celldiff.y))
v_xy_sets[key].add(int(celldiff.v))
v2_xy_sets[key].add(int(celldiff.v_2))
assert len(v_xy_sets) == len(v2_xy_sets)
for xy in v_xy_sets:
assert v_xy_sets[xy] == v2_xy_sets[xy], \
"Synthetic dimension trouble at {0}".format(xy)
def get_instance_info():
""" Obtain server/port information for all running scidb instances.
Data is returned as a list of namedtuples with the same attributes
as in "list('instances')" output.
"""
return make_table("InstanceInfo",
"list('instances')",
host=os.environ.get('IQUERY_HOST', '127.0.0.1'),
port=os.environ.get('IQUERY_PORT', '1239'))
def get_chunk_statistics(array_name, _arrays=[], _chdescs=[]):
"""Derive chunk statistics for the given array."""
# Read and cache results of these list() queries.
if not _arrays:
_arrays = make_table('array', "list('arrays', true)")
if not _chdescs:
want = "arrid,attid,nelem,csize,usize,asize"
_chdescs = make_table('chdesc',
"project(list('chunk descriptors'),%s)" % want)
# Find the versioned array id (vaid) of the given array name.
if '@' in array_name:
aname, ver = array_name.split('@')
ver = int(ver)
pairs = [(ver, int(x.aid)) for x in _arrays if x.name == array_name]
else:
aname = array_name
pairs = [(int(x.name[x.name.index('@') + 1:]), int(x.aid))
for x in _arrays if x.name.startswith("%s@" % array_name)]
if not pairs:
raise AppError("Array '{0}' not found".format(array_name))
vaid = sorted(pairs)[-1][1]
dbg("Versioned array id for", array_name, "is", vaid)
# Use vaid to collect chunk data from the chunkdesc table.
attr_table = make_table('attrs', "attributes(%s)" % aname)
attr2name = dict([(int(x.No), x.name) for x in attr_table])
attr2chunks = dict([(int(x.No), []) for x in attr_table])
ebm = len(attr2name)
attr2name[ebm] = "emptyBitmap"
attr2chunks[ebm] = []
nchunks = 0
chunks_per_attr = Counter()
for cdesc in _chdescs:
if int(cdesc.arrid) == vaid:
aid = int(cdesc.attid)
nchunks += 1
chunks_per_attr[aid] += 1
# Appended list *MUST* be in same order as CHUNK_STAT_TUPLES.
attr2chunks[aid].append(
map(int, (cdesc.nelem, cdesc.csize, cdesc.usize, cdesc.asize)))
# Paranoid check that all attributes have same number of chunks.
expected_cpa = nchunks // len(attr2name)
complaints = []
for x in chunks_per_attr:
if chunks_per_attr[x] != expected_cpa:
complaints.append("Attribute {0} has unexpected chunk count {1},"
" should be {2}".format(x, chunks_per_attr[x],
expected_cpa))
if complaints:
raise AppError('\n'.join(complaints))
# Build ChunkStats objects, one per attribute.
all_chunk_stats = []
for aid in attr2name:
if _PPRINT_FRIENDLY:
stats = [
array_name, (attr2name[aid], aid),
('chunks_per_attr', chunks_per_attr[aid])
]
else:
stats = [array_name, attr2name[aid], aid, chunks_per_attr[aid]]
for i, datum in enumerate(CHUNK_STAT_TUPLES):
data = sorted([x[i] for x in attr2chunks[aid]])
stat = Stat(StatLib.mean(data), StatLib.stdev(data), data[0],
StatLib.median(data), data[-1])
if _PPRINT_FRIENDLY:
stats.append((datum, stat))
else:
stats.append(stat)
if _PPRINT_FRIENDLY:
all_chunk_stats.append(stats)
else:
all_chunk_stats.append(ChunkStats(*stats))
# Done!
return all_chunk_stats