def test_iter_interval(self):
visparam = paramcontainer.VisParamContainer(vis=self.vis)
reader = visreader.VisibilityReader(visparam)
chunk_id = 0
nrow_total = 0
interval = 32
for chunk in reader.readvis(items=['time'], interval=interval):
nrow = chunk['time'].shape[-1]
print('LOG chunk {0}: nrow={1} items={2} itemshape={3}'.format(
chunk_id, nrow, list(chunk.keys()), chunk['time'].shape))
self.verify_chunk_consistency(chunk, chunk_id)
chunk_id += 1
nrow_total += nrow
times = chunk['time']
mintimeidx = times.argmin()
maxtimeidx = times.argmax()
timemin = times[mintimeidx]
timemax = times[maxtimeidx]
time_interval = timemax - timemin
self.assertLessEqual(time_interval, interval)
nrow_total_expected = self._expected_nrows()
print(
'LOG number of rows for input ms {0}'.format(nrow_total_expected))
self.assertEqual(nrow_total_expected, nrow_total)
def test_iter_columns(self):
visparam = paramcontainer.VisParamContainer(vis=self.vis)
reader = visreader.VisibilityReader(visparam)
chunk_id = 0
nrow_total = 0
columns = ['ANTENNA1']
items = ['antenna1']
antenna_list = []
for chunk in reader.readvis(items=items,
columns=columns,
adddefault=False):
nrow = chunk[items[0]].shape[-1]
print('LOG chunk {0}: nrow={1} items={2} itemshape={3}'.format(
chunk_id, nrow, list(chunk.keys()), chunk[items[0]].shape))
self.verify_chunk_consistency(chunk, chunk_id)
chunk_id += 1
nrow_total += nrow
antenna = chunk['antenna1']
self.assertTrue(numpy.all(antenna == antenna[0]))
self.assertFalse(antenna[0] in antenna_list)
antenna_list.append(antenna[0])
nrow_total_expected = self._expected_nrows()
print(
'LOG number of rows for input ms {0}'.format(nrow_total_expected))
self.assertEqual(nrow_total_expected, nrow_total)
def test_items(self):
visparam = paramcontainer.VisParamContainer(vis=self.vis)
reader = visreader.VisibilityReader(visparam)
chunk_id = 0
nrow_total = 0
items = ['antenna1', 'antenna2', 'uvw', 'time']
for chunk in reader.readvis(items=items):
nrow = chunk[items[0]].shape[-1]
print('LOG chunk {0}: nrow={1} items={2} itemshape={3}'.format(
chunk_id, nrow, list(chunk.keys()),
list(chunk.values())[0].shape))
self.verify_chunk_consistency(chunk, chunk_id)
chunk_id += 1
nrow_total += nrow
# chunk has additional field 'chunk_id'
self.assertEqual(len(chunk), len(items) + 1)
for item in items:
self.assertTrue(item in chunk)
nrow_total_expected = self._expected_nrows()
print(
'LOG number of rows for input ms {0}'.format(nrow_total_expected))
self.assertEqual(nrow_total_expected, nrow_total)
def test_msselect(self):
visparam = paramcontainer.VisParamContainer(vis=self.vis, spw='1')
reader = visreader.VisibilityReader(visparam)
chunk_id = 0
nrow_total = 0
for chunk in reader.readvis(items=['data_desc_id']):
nrow = chunk['data_desc_id'].shape[-1]
print('LOG chunk {0}: nrow={1} items={2} itemshape={3}'.format(
chunk_id, nrow, list(chunk.keys()),
chunk['data_desc_id'].shape))
self.verify_chunk_consistency(chunk, chunk_id)
chunk_id += 1
nrow_total += nrow
self.assertTrue('data_desc_id' in chunk)
dd = chunk['data_desc_id']
self.assertTrue(numpy.all(dd == 1))
nrow_total_expected = self._expected_nrows('DATA_DESC_ID==1')
print(
'LOG number of rows for input ms {0}'.format(nrow_total_expected))
self.assertEqual(nrow_total_expected, nrow_total)
def test_iter(self):
visparam = paramcontainer.VisParamContainer(vis=self.vis)
reader = visreader.VisibilityReader(visparam)
nrow_total = 0
chunk_id = 0
for chunk in reader.readvis(
items=['time', 'data_desc_id', 'data', 'weight']):
nrow = chunk['time'].shape[-1]
print('LOG chunk {0}: nrow={1} items={2} itemshape={3}'.format(
chunk_id, nrow, list(chunk.keys()),
list(chunk.values())[0].shape))
self.verify_chunk_consistency(chunk, chunk_id)
nrow_total += nrow
chunk_id += 1
data_list = chunk['data']
ddid_list = chunk['data_desc_id']
weight_list = chunk['weight']
for irow in range(nrow):
data = data_list[..., irow]
weight = weight_list[..., irow]
ddid = ddid_list[irow]
nchan = self.nchanmap[ddid]
ncorr = self.ncorrmap[ddid]
self.assertEqual(data.shape, (
ncorr,
nchan,
))
self.assertEqual(weight.shape, (ncorr, ))
print('LOG iterated {0} times in total'.format(chunk_id))
print('LOG total number of rows {0}'.format(nrow_total))
nrow_total_expected = self._expected_nrows()
print(
'LOG number of rows for input ms {0}'.format(nrow_total_expected))
self.assertEqual(nrow_total_expected, nrow_total)
def test_msselect_chan(self):
self.skipTest('Channel selection is not effective to ms iterator')
visparam = paramcontainer.VisParamContainer(vis=self.vis, spw='0:0~9')
reader = visreader.VisibilityReader(visparam)
chunk_id = 0
nrow_total = 0
for chunk in reader.readvis(items=['time', 'data_desc_id', 'data']):
nrow = chunk['time'].shape[-1]
print('LOG chunk {0}: nrow={1} items={2} itemshape={3}'.format(
chunk_id, nrow, list(chunk.keys()),
list(chunk.values())[0].shape))
self.verify_chunk_consistency(chunk, chunk_id)
chunk_id += 1
nrow_total += nrow
data_list = chunk['data']
ddid_list = chunk['data_desc_id']
for irow in range(nrow):
data = data_list[:, :, irow]
ddid = ddid_list[irow]
self.assertEqual(ddid, 0)
nchan = min(self.nchanmap[ddid], 10)
print('nchan = {0}'.format(nchan))
ncorr = self.ncorrmap[ddid]
self.assertEqual(data.shape, (
ncorr,
nchan,
))
print('LOG iterated {0} times in total'.format(chunk_id))
print('LOG total number of rows {0}'.format(nrow_total))
nrow_total_expected = self._expected_nrows('DATA_DESC_ID==0')
print(
'LOG number of rows for input ms {0}'.format(nrow_total_expected))
self.assertEqual(nrow_total_expected, nrow_total)
def test_iter_nrow(self):
visparam = paramcontainer.VisParamContainer(vis=self.vis)
reader = visreader.VisibilityReader(visparam)
chunk_id = 0
nrow_total = 0
nrow_chunk = 4
for chunk in reader.readvis(nrow=nrow_chunk):
nrow = chunk['time'].shape[-1]
print('LOG chunk {0}: nrow={1} items={2} itemshape={3}'.format(
chunk_id, nrow, list(chunk.keys()),
list(chunk.values())[0].shape))
self.verify_chunk_consistency(chunk, chunk_id)
chunk_id += 1
nrow_total += nrow
self.assertLessEqual(nrow, nrow_chunk)
nrow_total_expected = self._expected_nrows()
print(
'LOG number of rows for input ms {0}'.format(nrow_total_expected))
self.assertEqual(nrow_total_expected, nrow_total)
def test_parallel(self):
# create reader
reader = visreader.VisibilityReader(self.visparam)
# create converter
start = 3
width = 10
nchan = 1
converter = self.get_converter(start, width, nchan)
max_nrow = 16
num_measure = 10
serial_results = numpy.empty(num_measure, dtype=numpy.float32)
parallel_results = numpy.empty_like(serial_results)
conv = lambda chunk: (chunk['chunk_id'],
converter.generate_working_set(chunk))
for i in range(num_measure):
# serial run
ws_serial = []
start_serial = time.time()
for chunk_id, ws in map(conv, reader.readvis(nrow=max_nrow)):
ws_serial.append((chunk_id, ws))
end_serial = time.time()
serial_results[i] = end_serial - start_serial
print('SERIAL RUN: {0} sec'.format(serial_results[i]))
# parallel run
num_threads = 2
ws_parallel = []
start_parallel = time.time()
for chunk_id, ws in sakura.paraMap(num_threads, conv,
reader.readvis(nrow=max_nrow)):
ws_parallel.append((chunk_id, ws))
end_parallel = time.time()
parallel_results[i] = end_parallel - start_parallel
print('PARALLEL RUN: {0} sec'.format(parallel_results[i]))
# consistency check
for chunk_id, ws in ws_serial:
self.verify_ws_consistency(ws, chunk_id)
for chunk_id, ws in ws_parallel:
self.verify_ws_consistency(ws, chunk_id)
self.assertEqual(len(ws_serial), len(ws_parallel))
for _, p in ws_parallel:
found = False
for i in range(len(ws_serial)):
_, s = ws_serial[i]
if p.data_id == s.data_id:
_, s = ws_serial.pop(i)
self.assertTrue(numpy.all(p.u == s.u))
self.assertTrue(numpy.all(p.v == s.v))
self.assertTrue(numpy.all(p.rdata == s.rdata))
self.assertTrue(numpy.all(p.idata == s.idata))
self.assertTrue(numpy.all(p.flag == s.flag))
self.assertTrue(numpy.all(p.row_flag == s.row_flag))
self.assertTrue(numpy.all(p.weight == s.weight))
self.assertTrue(
numpy.all(p.channel_map == s.channel_map))
found = True
break
self.assertTrue(found)
# verification
print('LOG: SERIAL RUN: {0}'.format(serial_results))
print('LOG: PARALLEL RUN: {0}'.format(parallel_results))
acceleration = serial_results / parallel_results
print('LOG: ACCELERATION: {0} (max {1} min {2})'.format(
acceleration, acceleration.max(), acceleration.min()))
self.assertLess(parallel_results.min(), serial_results.min())