def transmit():
print 'TX: initializing'
tx = spead.Transmitter(spead.TransportUDPtx('127.0.0.1', PORT))
ig = spead.ItemGroup()
print 'TX start'
for i in range(100):
ig.add_item(name='var%d' % i,
description='Description for var%d' % i,
init_val=0)
ig.add_item(name='tx_time', description='Description', fmt='f\x00\x00\x40')
ig.add_item(name='pv_time', description='Description', fmt='f\x00\x00\x40')
ig.add_item(name='data',
description='Description for data',
shape=SHAPE,
fmt='i\x00\x00\x20')
data0 = numpy.zeros(SHAPE)
data1 = numpy.ones(SHAPE)
ig['pv_time'] = time.time()
for i in range(20):
ig['var%d' % i] = 1
if i % 2 == 0: ig['data'] = data0
else: ig['data'] = data1
ig['tx_time'] = time.time()
tx.send_heap(ig.get_heap())
ig['pv_time'] = time.time()
t_tx = ig['pv_time'] - ig['tx_time']
print 't_tx:', t_tx
tx.end()
print 'TX stop'
def transmit():
print 'TX: Initializing...'
tx = spead.Transmitter(spead.TransportUDPtx('127.0.0.1', PORT))
ig = spead.ItemGroup()
ig.add_item(name='Var1',
description='Description for Var1',
shape=[],
fmt=spead.mkfmt(('u', 32), ('u', 32), ('u', 32)),
init_val=(1, 2, 3))
tx.send_heap(ig.get_heap())
ig['Var1'] = (4, 5, 6)
tx.send_heap(ig.get_heap())
ig.add_item(name='Var2',
description='Description for Var2',
shape=[100, 100],
fmt=spead.mkfmt(('u', 32)))
data = numpy.arange(100 * 100)
data.shape = (100, 100)
ig['Var2'] = data
tx.send_heap(ig.get_heap())
tx.end()
print 'TX: Done.'
def transmit_std():
print 'TX: Initializing standard transport TX to IP', options.ip
tx = spead.Transmitter(spead.TransportUDPtx(options.ip, 8888))
ig = spead.ItemGroup()
ig.add_item(
name='data_timestamp',
description='Timestamp in epoch ms for the current visibility sample',
shape=[1],
fmt=spead.mkfmt(('u', 64)))
ig.add_item(
name='vis_data',
description='The complex visibility spectrum for a single time dump',
shape=[channels, baselines, 2],
fmt=spead.mkfmt(('u', 32)))
# using init_val with a numpy array will use the numpy transport automatically.
t_heap_send = 0
for x in range(iterations):
ig['data_timestamp'] = int(time.time() * 1000)
ig['vis_data'] = np.random.normal(size=(channels, baselines,
2)).astype(np.float32)
t_heap_send = time.time()
tx.send_heap(ig.get_heap())
print "Sent data for timestamp", ig['data_timestamp'], "in", time.time(
) - t_heap_send, "s"
time.sleep(15)
tx.end()
print 'TX: Done.'
def send():
tx = spead.Transmitter(spead.TransportUDPtx('127.0.0.1', PORT))
ig = spead.ItemGroup()
ig.add_item(name="var1", description="this is a variable", init_val=333)
data0 = numpy.ones(SHAPE)
ig.add_item(name="data1",
description="these are some ones",
shape=SHAPE,
fmt='i\x00\x00\x20',
init_val=data0)
#ig.add_item(name="data2", description="these are some zeros", shape=SHAPE, fmt='i\x00\x00\x20', init_val=data0)
#ig.add_item(name="data3", description="these are some zeros", shape=SHAPE, fmt='i\x00\x00\x20', init_val=data0)
#ig.add_item(name="data4", description="these are some zeros", shape=SHAPE, fmt='i\x00\x00\x20', init_val=data0)
tx.send_heap(ig.get_heap())
tx.end()
def transmit():
# Initialise SPEAD transmitter
tx = spead.Transmitter(spead.TransportUDPtx(IP, PORT))
# Add items to item group
ig = spead.ItemGroup()
ig.add_item(name="station", shape=[], fmt=spead.mkfmt(('u', 1)), init_val = (1))
data = numpy.arange(40000).astype(numpy.uint32); data.shape = (40000)
ig.add_item(name='data', shape=[40000], ndarray=data)
# Continuously send heap
while True:
tx.send_heap(ig.get_heap())
tx.end()
print 'TX: Done.'
def rx_cont(self,data_port=7148, sd_ip='127.0.0.1', sd_port=7149,acc_scale=True, filename=None,**kwargs):
logger=self.logger
logger.info("Data reception on port %i."%data_port)
rx = spead.TransportUDPrx(data_port, pkt_count=1024, buffer_size=51200000)
logger.info("Sending Signal Display data to %s:%i."%(sd_ip,sd_port))
tx_sd = spead.Transmitter(spead.TransportUDPtx(sd_ip, sd_port))
ig = spead.ItemGroup()
ig_sd = spead.ItemGroup()
if filename == None:
filename=str(int(time.time())) + ".synth.h5"
logger.info("Starting file %s."%(filename))
f = h5py.File(filename, mode="w")
data_ds = None
ts_ds = None
idx = 0
dump_size = 0
datasets = {}
datasets_index = {}
meta_required = ['n_chans','bandwidth','n_bls','n_xengs','center_freq','bls_ordering']
# we need these bits of meta data before being able to assemble and transmit signal display data
meta_desired = ['n_accs']
meta = {}
for heap in spead.iterheaps(rx):
ig.update(heap)
logger.debug("PROCESSING HEAP idx(%i) cnt(%i) @ %.4f" % (idx, heap.heap_cnt, time.time()))
for name in ig.keys():
item = ig.get_item(name)
if not item._changed and datasets.has_key(name): continue # the item is not marked as changed, and we have a record for it
if name in meta_desired:
meta[name] = ig[name]
if name in meta_required:
meta[name] = ig[name]
meta_required.pop(meta_required.index(name))
if len(meta_required) == 0:
#sd_frame = np.zeros((meta['n_chans'],meta['n_bls'],2),dtype=np.float32)
logger.info("Got all required metadata. Expecting data frame shape of %i %i %i"%(meta['n_chans'],meta['n_bls'],2))
meta_required = ['n_chans','bandwidth','n_bls','n_xengs','center_freq','bls_ordering']
ig_sd = spead.ItemGroup()
for meta_item in meta_required:
ig_sd.add_item(
name=ig.get_item(meta_item).name,
id=ig.get_item(meta_item).id,
description=ig.get_item(meta_item).description,
#shape=ig.get_item(meta_item).shape,
#fmt=ig.get_item(meta_item).format,
init_val=ig.get_item(meta_item).get_value())
tx_sd.send_heap(ig_sd.get_heap())
if not datasets.has_key(name):
# check to see if we have encountered this type before
shape = ig[name].shape if item.shape == -1 else item.shape
dtype = np.dtype(type(ig[name])) if shape == [] else item.dtype
if dtype is None: dtype = ig[name].dtype
# if we can't get a dtype from the descriptor try and get one from the value
logger.info("Creating dataset for %s (%s,%s)."%(str(name),str(shape),str(dtype)))
f.create_dataset(name,[1] + ([] if list(shape) == [1] else list(shape)), maxshape=[None] + ([] if list(shape) == [1] else list(shape)), dtype=dtype)
dump_size += np.multiply.reduce(shape) * dtype.itemsize
datasets[name] = f[name]
datasets_index[name] = 0
if not item._changed: continue
# if we built from and empty descriptor
else:
logger.info("Adding %s to dataset. New size is %i."%(name,datasets_index[name]+1))
f[name].resize(datasets_index[name]+1, axis=0)
if name.startswith("xeng_raw"):
sd_timestamp = ig['sync_time'] + (ig['timestamp'] / float(ig['scale_factor_timestamp']))
#logger.info("SD Timestamp: %f (%s)."%(sd_timestamp,time.ctime(sd_timestamp)))
scale_factor=float(meta['n_accs'] if (meta.has_key('n_accs') and acc_scale) else 1)
scaled_data = (ig[name]/scale_factor).astype(np.float32)
# reinit the group to force meta data resend
ig_sd = spead.ItemGroup()
ig_sd.add_item(name=('sd_data'),
id=(0x3501),
description="Combined raw data from all x engines.",
ndarray=(scaled_data.dtype,scaled_data.shape))
ig_sd.add_item(name=('sd_timestamp'),
id=0x3502,
description='Timestamp of this sd frame in centiseconds since epoch (40 bit limitation).',
init_val=sd_timestamp)
#shape=[],
#fmt=spead.mkfmt(('u',spead.ADDRSIZE)))
t_it = ig_sd.get_item('sd_data')
logger.debug("Added SD frame with shape %s, dtype %s"%(str(t_it.shape),str(t_it.dtype)))
tx_sd.send_heap(ig_sd.get_heap())
logger.info("Sending signal display frame with timestamp %i (%s). %s. Max: %i, Mean: %i"%(
sd_timestamp,
time.ctime(sd_timestamp),
"Unscaled" if not acc_scale else "Scaled by %i" % (scale_factor),
np.max(scaled_data),
np.mean(scaled_data)))
ig_sd['sd_data'] = scaled_data
ig_sd['sd_timestamp'] = sd_timestamp * 100
#ig_sd['sd_timestamp'] = sd_timestamp
tx_sd.send_heap(ig_sd.get_heap())
f[name][datasets_index[name]] = ig[name]
datasets_index[name] += 1
item._changed = False
# we have dealt with this item so continue...
idx+=1
# for (name,idx) in datasets_index.iteritems():
# if idx == 1:
# self.logger.info("Repacking dataset %s as an attribute as it is singular."%name)
# f['/'].attrs[name] = f[name].value[0]
# f.__delitem__(name)
logger.info("Got a SPEAD end-of-stream marker. Closing File.")
f.flush()
f.close()
rx.stop()
ig_sd = None
sd_timestamp = None
logger.info("Files and sockets closed.")
def rx_inter(self,data_port=7148, sd_ip='127.0.0.1', sd_port=7149, acc_scale=True, filename=None, **kwargs):
'''
Process SPEAD data from X engines and forward it to the SD.
'''
print 'WARNING: This function is not yet tested. YMMV.'
logger=self.logger
logger.info("Data reception on port %i."%data_port)
rx = spead.TransportUDPrx(data_port, pkt_count=1024, buffer_size=51200000)
logger.info("Sending Signal Display data to %s:%i."%(sd_ip,sd_port))
tx_sd = spead.Transmitter(spead.TransportUDPtx(sd_ip, sd_port))
ig = spead.ItemGroup()
ig_sd = spead.ItemGroup()
if filename == None:
filename=str(int(time.time())) + ".synth.h5"
logger.info("Starting file %s."%(filename))
f = h5py.File(filename, mode="w")
data_ds = None
ts_ds = None
idx = 0
dump_size = 0
datasets = {}
datasets_index = {}
# we need these bits of meta data before being able to assemble and transmit signal display data
meta_required = ['n_chans','n_bls','n_xengs','center_freq','bls_ordering','bandwidth']
meta_desired = ['n_accs']
meta = {}
sd_frame = None
sd_slots = None
timestamp = None
# log the latest timestamp for which we've stored data
currentTimestamp = -1
# iterate through SPEAD heaps returned by the SPEAD receiver.
for heap in spead.iterheaps(rx):
ig.update(heap)
logger.debug("PROCESSING HEAP idx(%i) cnt(%i) @ %.4f" % (idx, heap.heap_cnt, time.time()))
for name in ig.keys():
item = ig.get_item(name)
# the item is not marked as changed and we already have a record for it, continue
if not item._changed and datasets.has_key(name):
continue
logger.debug("PROCESSING KEY %s @ %.4f" % (name, time.time()))
if name in meta_desired:
meta[name] = ig[name]
if name in meta_required:
meta[name] = ig[name]
meta_required.pop(meta_required.index(name))
if len(meta_required) == 0:
sd_frame = np.zeros((meta['n_chans'],meta['n_bls'],2),dtype=np.float32)
logger.info("Got all required metadata. Initialised sd frame to shape %s"%(str(sd_frame.shape)))
meta_required = ['n_chans','bandwidth','n_bls','n_xengs','center_freq','bls_ordering']
ig_sd = spead.ItemGroup()
for meta_item in meta_required:
ig_sd.add_item(
name=ig.get_item(meta_item).name,
id=ig.get_item(meta_item).id,
description=ig.get_item(meta_item).description,
#shape=ig.get_item(meta_item).shape,
#fmt=ig.get_item(meta_item).format,
init_val=ig.get_item(meta_item).get_value())
tx_sd.send_heap(ig_sd.get_heap())
sd_slots = np.zeros(meta['n_xengs'])
if not datasets.has_key(name):
# check to see if we have encountered this type before
shape = ig[name].shape if item.shape == -1 else item.shape
dtype = np.dtype(type(ig[name])) if shape == [] else item.dtype
if dtype is None: dtype = ig[name].dtype
# if we can't get a dtype from the descriptor, try and get one from the value
logger.info("Creating dataset for %s (%s,%s)."%(str(name),str(shape),str(dtype)))
f.create_dataset(name,[1] + ([] if list(shape) == [1] else list(shape)), maxshape=[None] + ([] if list(shape) == [1] else list(shape)), dtype=dtype)
dump_size += np.multiply.reduce(shape) * dtype.itemsize
datasets[name] = f[name]
datasets_index[name] = 0
# if we built from an empty descriptor
if not item._changed:
continue
else:
logger.info("Adding %s to dataset. New size is %i."%(name,datasets_index[name]+1))
f[name].resize(datasets_index[name]+1, axis=0)
# now we store this x engine's data for sending sd data.
if sd_frame is not None and name.startswith("xeng_raw"):
xeng_id = int(name[8:])
sd_frame[xeng_id::meta['n_xengs']] = ig[name]
logger.debug('Received data for Xeng %i @ %.4f' % (xeng_id, time.time()))
# we got a timestamp.
if sd_frame is not None and name.startswith("timestamp"):
xeng_id = int(name[9:])
timestamp = ig['sync_time'] + (ig[name] / ig['scale_factor_timestamp']) #in seconds since unix epoch
localTime = time.time()
print "Decoded timestamp for Xeng", xeng_id, ":", timestamp, " (", time.ctime(timestamp),") @ %.4f" % localTime, " ", time.ctime(localTime), "diff(", localTime-timestamp, ")"
# is this timestamp in the past?
if currentTimestamp > timestamp:
errorString = "Timestamp %.2f (%s) is earlier than the current timestamp %.2f (%s). Ignoring..." % (timestamp, time.ctime(timestamp), currentTimestamp, time.ctime(currentTimestamp))
logger.warning(errorString)
continue
# is this a new timestamp before a complete set?
if (timestamp > currentTimestamp) and sd_slots.any():
errorString = "New timestamp %.2f from Xeng%i before previous set %.2f sent" % (timestamp, xeng_id, currentTimestamp)
logger.warning(errorString)
sd_slots = np.zeros(meta['n_xengs'])
sd_frame = np.zeros((ig['n_chans'],ig['n_bls'],2),dtype=sd_frame.dtype)
currentTimestamp = -1
continue
# is this new timestamp in the past for this X engine?
if timestamp <= sd_slots[xeng_id]:
errorString = 'Xeng%i already on timestamp %.2f but got %.2f now, THIS SHOULD NOT HAPPEN' % (xeng_id, sd_slots[xeng_id], timestamp)
logger.error(errorString)
raise RuntimeError(errorString)
# update our info on which integrations we have
sd_slots[xeng_id] = timestamp
currentTimestamp = timestamp
# do we have integration data and timestamps for all the xengines? If so, send the SD frame.
if timestamp is not None and sd_frame is not None and sd_slots is not None and sd_slots.all():
ig_sd = spead.ItemGroup()
# make sure we have the right dtype for the sd data
ig_sd.add_item(name=('sd_data'), id=(0x3501), description="Combined raw data from all x engines.", ndarray=(sd_frame.dtype,sd_frame.shape))
ig_sd.add_item(name=('sd_timestamp'), id=0x3502, description='Timestamp of this sd frame in centiseconds since epoch (40 bit limitation).', shape=[], fmt=spead.mkfmt(('u',spead.ADDRSIZE)))
t_it = ig_sd.get_item('sd_data')
logger.info("Added SD frame with shape %s, dtype %s" % (str(t_it.shape),str(t_it.dtype)))
scale_factor=(meta['n_accs'] if meta.has_key('n_accs') else 1)
logger.info("Sending signal display frame with timestamp %i (%s). %s. @ %.4f" % (timestamp, time.ctime(timestamp), "Unscaled" if not acc_scale else "Scaled by %i" % (scale_factor), time.time()))
ig_sd['sd_data'] = sd_frame.astype(np.float32) if not acc_scale else (sd_frame / float(scale_factor)).astype(np.float32)
ig_sd['sd_timestamp'] = int(timestamp * 100)
tx_sd.send_heap(ig_sd.get_heap())
# reset the arrays that hold integration data
sd_slots = np.zeros(meta['n_xengs'])
sd_frame = np.zeros((ig['n_chans'],ig['n_bls'],2),dtype=sd_frame.dtype)
timestamp = None
f[name][datasets_index[name]] = ig[name]
datasets_index[name] += 1
item._changed = False
idx+=1
logger.info("Got a SPEAD end-of-stream marker. Closing File.")
f.flush()
f.close()
rx.stop()
sd_frame = None
sd_slots = None
ig_sd = None
elif ig['command'] == CMD_CACHE:
print 'caching data'
cal = ig['cal'].tostring()
if aa is None:
aa = a.cal.get_aa(cal, ig['sdf'], ig['sfreq'], ig['nchan'])
print ig['chans'].squeeze()
aa.select_chans(ig['chans'].squeeze())
aa.set_jultime(ig['juldate'])
if cat is None:
srclist, cutoff, catalogs = a.scripting.parse_srcs(
ig['src'].tostring(), ig['cat'].tostring())
cat = a.cal.get_catalog(cal, srclist, cutoff, catalogs)
cat.compute(aa)
mfq = cat.get('mfreq')
if ig_tx is None:
tport_tx = spead.TransportUDPtx(ig['respond_ip'].tostring(),
ig['respond_port'])
tx = spead.Transmitter(tport_tx)
ig_tx = spead.ItemGroup()
ig_tx.add_item('score', fmt=f64)
ig_tx.add_item('nsamples', fmt=f64)
ig_tx.add_item('valid', fmt=i40, init_val=0)
# Finally, cache this data
t = ig['juldate']
bl = a.miriad.ij2bl(*ig['baseline'])
pol = ''.join(ig['pol'][-2:])
d = ig['data_real'].squeeze() + 1j * ig['data_imag'].squeeze()
f = ig['flags'].squeeze()
print t, a.miriad.bl2ij(bl) #, d
if not dbuf.has_key(t): dbuf[t] = {}
if not dbuf[t].has_key(pol): dbuf[t][pol] = {}
dbuf[t][pol][bl] = (d, f, n.where(f, 0, n.abs(d)**2).sum())
# Parse command-line options
opts.ant += ',cross'
if opts.maxiter < 0: opts.maxiter = n.Inf
# Process daemons
def parsehostport(hostport):
host, port = hostport.split(':')
return (host, int(port))
daemon_ip_ports = [parsehostport(w) for w in opts.daemons.split(',')]
NDAEMONS = len(daemon_ip_ports)
_igs = [spead.ItemGroup() for i in range(NDAEMONS)]
_txs = [
spead.Transmitter(spead.TransportUDPtx(*ip_port))
for ip_port in daemon_ip_ports
]
_igrxs = [spead.ItemGroup() for i in range(NDAEMONS)]
_iters = [
spead.iterframes(spead.TransportUDPrx(opts.baserx + i))
for i in range(NDAEMONS)
]
def igs(key, value, num=None):
if num is None:
for ig in _igs:
ig[key] = value
else:
_igs[num][key] = value
i = f.get_items()
else:
ValueError(cmd)
elif cmd.startswith('BufferSocket'):
if cmd == 'BufferSocket':
while True:
b = BufferSocket(pkt_count=8192)
elif cmd.split('.')[1] == 'callback':
b = BufferSocket(pkt_count=8192)
def callback(pkt):
pass
b.set_callback(callback)
b.start(53000)
tx = S.TransportUDPtx('localhost', 53000)
while True:
tx.write(pkt)
elif cmd.startswith('pack'):
d = [[0]] * 1024 * 1024
while True:
s = pack('u\x00\x00\x20', d)
elif cmd.startswith('unpack'):
s = '\x00' * 1024
while True:
d = unpack('u\x00\x00\x20', s, cnt=-1)
elif cmd.startswith('ItemGroup'):
ig1, ig2 = S.ItemGroup(), S.ItemGroup()
ig1.add_item('var1', fmt='f\x00\x00\x40', shape=-1)
while True:
ig1['var1'] = [[0]] * 1024
def setUp(self):
self.t_tx = S.TransportUDPtx(ip='127.0.0.1', port=50000)
def setUp(self):
self.t_tx = S.TransportUDPtx(ip='127.0.0.1', port=50001)
self.t_rx = RawUDPrx(port=50001)