def __init__(
self,
channel_dir,
start=None,
end=None,
repeat=False,
gapless=False,
min_chunksize=None,
):
"""Read a channel of data from a Digital RF directory.
In addition to outputting samples from Digital RF format data, this
block also emits a 'properties' message containing inherent channel
properties and adds stream tags using the channel's accompanying
Digital Metadata. See the Notes section for details on what the
messages and stream tags contain.
Parameters
----------
channel_dir : string | list of strings
Either a single channel directory containing 'drf_properties.h5'
and timestamped subdirectories with Digital RF files, or a list of
such. A directory can be a file system path or a url, where the url
points to a channel directory. Each must be a local path, or start
with 'http://'', 'file://'', or 'ftp://''.
Other Parameters
----------------
start : None | int | float | string, optional
A value giving the start of the channel's playback.
If None or '', the start of the channel's available data is used.
If an integer, it is interpreted as a sample index given in the
number of samples since the epoch (time_since_epoch*sample_rate).
If a float, it is interpreted as a UTC timestamp (seconds since
epoch).
If a string, four forms are permitted:
1) a string which can be evaluated to an integer/float and
interpreted as above,
2) a string beginning with '+' and followed by an integer
(float) expression, interpreted as samples (seconds) from
the start of the data, and
3) a time in ISO8601 format, e.g. '2016-01-01T16:24:00Z'
4) 'now' ('nowish'), indicating the current time (rounded up)
end : None | int | float | string, optional
A value giving the end of the channel's playback.
If None or '', the end of the channel's available data is used.
See `start` for a description of how this value is interpreted.
repeat : bool, optional
If True, loop the data continuously from the start after the end
is reached. If False, stop after the data is read once.
gapless : bool, optional
If True, output default-filled samples for any missing data between
start and end. If False, skip missing samples and add an `rx_time`
stream tag to indicate the gap.
min_chunksize : None | int, optional
Minimum number of samples to output at once. This value can be used
to adjust the source's performance to reduce underruns and
processing time. If None, a sensible default will be used.
Notes
-----
A channel directory must contain subdirectories/files in the format:
[YYYY-MM-DDTHH-MM-SS]/rf@[seconds].[%03i milliseconds].h5
Each directory provided is considered the same channel. An error is
raised if their sample rates differ, or if their time periods overlap.
Upon start, this block sends a 'properties' message on its output
message port that contains a dictionary with one key, the channel's
name, and a value which is a dictionary of properties found in the
channel's 'drf_properties.h5' file.
This block emits the following stream tags at the appropriate sample
for each of the channel's accompanying Digital Metadata samples:
rx_time : (int secs, float frac) tuple
Time since epoch of the sample.
rx_rate : float
Sample rate in Hz.
rx_freq : float | 1-D array of floats
Center frequency or frequencies of the subchannels based on
the 'center_frequencies' metadata field.
metadata : dict
Any additional Digital Metadata fields are added to this
dictionary tag of metadata.
"""
if isinstance(channel_dir, six.string_types):
channel_dir = [channel_dir]
# eventually, we should re-factor DigitalRFReader and associated so
# that reading from a list of channel directories is possible
# with a DigitalRFChannelReader class or similar
# until then, split the path and use existing DigitalRFReader
top_level_dirs = []
chs = set()
for ch_dir in channel_dir:
top_level_dir, ch = os.path.split(ch_dir)
top_level_dirs.append(top_level_dir)
chs.add(ch)
if len(chs) == 1:
ch = chs.pop()
else:
raise ValueError("Channel directories must have the same name.")
self._ch = ch
self._Reader = DigitalRFReader(top_level_dirs)
self._properties = self._Reader.get_properties(self._ch)
typeclass = self._properties["H5Tget_class"]
itemsize = self._properties["H5Tget_size"]
is_complex = self._properties["is_complex"]
vlen = self._properties["num_subchannels"]
sr = self._properties["samples_per_second"]
self._itemsize = itemsize
self._sample_rate = sr
self._sample_rate_pmt = pmt.from_double(float(sr))
# determine output signature from HDF5 type metadata
typedict = get_h5type(typeclass, itemsize, is_complex)
self._outtype = typedict["name"]
self._itemtype = typedict["dtype"]
self._missingvalue = np.zeros((), dtype=self._itemtype)
self._missingvalue[()] = typedict["missingvalue"]
self._fillvalue = np.zeros((), dtype=self._itemtype)
if np.issubdtype(self._itemtype, np.inexact) and np.isnan(
self._missingvalue):
self._ismissing = lambda a: np.isnan(a)
else:
self._ismissing = lambda a: a == self._missingvalue
if vlen == 1:
out_sig = [self._itemtype]
else:
out_sig = [(self._itemtype, vlen)]
gr.sync_block.__init__(self,
name="digital_rf_channel_source",
in_sig=None,
out_sig=out_sig)
self.message_port_register_out(pmt.intern("properties"))
self._id = pmt.intern(self._ch)
self._tag_queue = {}
self._start = start
self._end = end
self._repeat = repeat
self._gapless = gapless
if min_chunksize is None:
# FIXME: it shouldn't have to be quite this high
self._min_chunksize = int(sr)
else:
self._min_chunksize = min_chunksize
# reduce CPU usage and underruns by setting a minimum number of samples
# to handle at once
# (really want to set_min_noutput_items, but no way to do that from
# Python)
try:
self.set_output_multiple(self._min_chunksize)
except RuntimeError:
traceback.print_exc()
errstr = "Failed to set source block min_chunksize to {min_chunksize}."
if min_chunksize is None:
errstr += (
" This value was calculated automatically based on the sample rate."
" You may have to specify min_chunksize manually.")
raise ValueError(errstr.format(min_chunksize=self._min_chunksize))
try:
self._DMDReader = self._Reader.get_digital_metadata(self._ch)
except IOError:
self._DMDReader = None
class digital_rf_channel_source(gr.sync_block):
"""Source block for reading a channel of Digital RF data."""
def __init__(
self,
channel_dir,
start=None,
end=None,
repeat=False,
gapless=False,
min_chunksize=None,
):
"""Read a channel of data from a Digital RF directory.
In addition to outputting samples from Digital RF format data, this
block also emits a 'properties' message containing inherent channel
properties and adds stream tags using the channel's accompanying
Digital Metadata. See the Notes section for details on what the
messages and stream tags contain.
Parameters
----------
channel_dir : string | list of strings
Either a single channel directory containing 'drf_properties.h5'
and timestamped subdirectories with Digital RF files, or a list of
such. A directory can be a file system path or a url, where the url
points to a channel directory. Each must be a local path, or start
with 'http://'', 'file://'', or 'ftp://''.
Other Parameters
----------------
start : None | int | float | string, optional
A value giving the start of the channel's playback.
If None or '', the start of the channel's available data is used.
If an integer, it is interpreted as a sample index given in the
number of samples since the epoch (time_since_epoch*sample_rate).
If a float, it is interpreted as a UTC timestamp (seconds since
epoch).
If a string, four forms are permitted:
1) a string which can be evaluated to an integer/float and
interpreted as above,
2) a string beginning with '+' and followed by an integer
(float) expression, interpreted as samples (seconds) from
the start of the data, and
3) a time in ISO8601 format, e.g. '2016-01-01T16:24:00Z'
4) 'now' ('nowish'), indicating the current time (rounded up)
end : None | int | float | string, optional
A value giving the end of the channel's playback.
If None or '', the end of the channel's available data is used.
See `start` for a description of how this value is interpreted.
repeat : bool, optional
If True, loop the data continuously from the start after the end
is reached. If False, stop after the data is read once.
gapless : bool, optional
If True, output default-filled samples for any missing data between
start and end. If False, skip missing samples and add an `rx_time`
stream tag to indicate the gap.
min_chunksize : None | int, optional
Minimum number of samples to output at once. This value can be used
to adjust the source's performance to reduce underruns and
processing time. If None, a sensible default will be used.
Notes
-----
A channel directory must contain subdirectories/files in the format:
[YYYY-MM-DDTHH-MM-SS]/rf@[seconds].[%03i milliseconds].h5
Each directory provided is considered the same channel. An error is
raised if their sample rates differ, or if their time periods overlap.
Upon start, this block sends a 'properties' message on its output
message port that contains a dictionary with one key, the channel's
name, and a value which is a dictionary of properties found in the
channel's 'drf_properties.h5' file.
This block emits the following stream tags at the appropriate sample
for each of the channel's accompanying Digital Metadata samples:
rx_time : (int secs, float frac) tuple
Time since epoch of the sample.
rx_rate : float
Sample rate in Hz.
rx_freq : float | 1-D array of floats
Center frequency or frequencies of the subchannels based on
the 'center_frequencies' metadata field.
metadata : dict
Any additional Digital Metadata fields are added to this
dictionary tag of metadata.
"""
if isinstance(channel_dir, six.string_types):
channel_dir = [channel_dir]
# eventually, we should re-factor DigitalRFReader and associated so
# that reading from a list of channel directories is possible
# with a DigitalRFChannelReader class or similar
# until then, split the path and use existing DigitalRFReader
top_level_dirs = []
chs = set()
for ch_dir in channel_dir:
top_level_dir, ch = os.path.split(ch_dir)
top_level_dirs.append(top_level_dir)
chs.add(ch)
if len(chs) == 1:
ch = chs.pop()
else:
raise ValueError("Channel directories must have the same name.")
self._ch = ch
self._Reader = DigitalRFReader(top_level_dirs)
self._properties = self._Reader.get_properties(self._ch)
typeclass = self._properties["H5Tget_class"]
itemsize = self._properties["H5Tget_size"]
is_complex = self._properties["is_complex"]
vlen = self._properties["num_subchannels"]
sr = self._properties["samples_per_second"]
self._itemsize = itemsize
self._sample_rate = sr
self._sample_rate_pmt = pmt.from_double(float(sr))
# determine output signature from HDF5 type metadata
typedict = get_h5type(typeclass, itemsize, is_complex)
self._outtype = typedict["name"]
self._itemtype = typedict["dtype"]
self._missingvalue = np.zeros((), dtype=self._itemtype)
self._missingvalue[()] = typedict["missingvalue"]
self._fillvalue = np.zeros((), dtype=self._itemtype)
if np.issubdtype(self._itemtype, np.inexact) and np.isnan(
self._missingvalue):
self._ismissing = lambda a: np.isnan(a)
else:
self._ismissing = lambda a: a == self._missingvalue
if vlen == 1:
out_sig = [self._itemtype]
else:
out_sig = [(self._itemtype, vlen)]
gr.sync_block.__init__(self,
name="digital_rf_channel_source",
in_sig=None,
out_sig=out_sig)
self.message_port_register_out(pmt.intern("properties"))
self._id = pmt.intern(self._ch)
self._tag_queue = {}
self._start = start
self._end = end
self._repeat = repeat
self._gapless = gapless
if min_chunksize is None:
# FIXME: it shouldn't have to be quite this high
self._min_chunksize = int(sr)
else:
self._min_chunksize = min_chunksize
# reduce CPU usage and underruns by setting a minimum number of samples
# to handle at once
# (really want to set_min_noutput_items, but no way to do that from
# Python)
try:
self.set_output_multiple(self._min_chunksize)
except RuntimeError:
traceback.print_exc()
errstr = "Failed to set source block min_chunksize to {min_chunksize}."
if min_chunksize is None:
errstr += (
" This value was calculated automatically based on the sample rate."
" You may have to specify min_chunksize manually.")
raise ValueError(errstr.format(min_chunksize=self._min_chunksize))
try:
self._DMDReader = self._Reader.get_digital_metadata(self._ch)
except IOError:
self._DMDReader = None
def _queue_tags(self, sample, tags):
"""Queue stream tags to be attached to data in the work function.
In addition to the tags specified in the `tags` dictionary, this will
add `rx_time` and `rx_rate` tags giving the sample time and rate.
Parameters
----------
sample : int
Sample index for the sample to tag, given in the number of samples
since the epoch (time_since_epoch*sample_rate).
tags : dict
Dictionary containing the tags to add with keys specifying the tag
name. The value is cast as an appropriate pmt type, while the name
will be turned into a pmt string in the work function.
"""
# add to current queued tags for sample if applicable
tag_dict = self._tag_queue.get(sample, {})
if not tag_dict:
# add time and rate tags
time = sample / self._sample_rate
tag_dict["rx_time"] = pmt.make_tuple(
pmt.from_uint64(int(np.uint64(time))),
pmt.from_double(float(time % 1)))
tag_dict["rx_rate"] = self._sample_rate_pmt
for k, v in tags.items():
try:
pmt_val = pmt.to_pmt(v)
except ValueError:
traceback.print_exc()
errstr = (
"Can't add tag for '{0}' because its value of {1} failed"
" to convert to a pmt value.")
print(errstr.format(k, v))
else:
tag_dict[k] = pmt_val
self._tag_queue[sample] = tag_dict
def start(self):
self._bounds = self._Reader.get_bounds(self._ch)
self._start_sample = util.parse_identifier_to_sample(
self._start, self._sample_rate, self._bounds[0])
self._end_sample = util.parse_identifier_to_sample(
self._end, self._sample_rate, self._bounds[0])
if self._start_sample is None:
self._read_start_sample = self._bounds[0]
else:
self._read_start_sample = self._start_sample
# add default tags to first sample
self._queue_tags(self._read_start_sample, {})
# replace longdouble samples_per_second with float for pmt conversion
properties_message = self._properties.copy()
properties_message["samples_per_second"] = float(
properties_message["samples_per_second"])
self.message_port_pub(pmt.intern("properties"),
pmt.to_pmt({self._ch: properties_message}))
return super(digital_rf_channel_source, self).start()
def work(self, input_items, output_items):
out = output_items[0]
nsamples = len(out)
next_index = 0
# repeat reading until we succeed or return
while next_index < nsamples:
read_start = self._read_start_sample
# read_end is inclusive, hence the -1
read_end = self._read_start_sample + (nsamples - next_index) - 1
# creating a read function that has an output argument so data can
# be copied directly would be nice
# also should move EOFError checking into reader once watchdog
# bounds functionality is implemented
try:
if self._end_sample is None:
if read_end > self._bounds[1]:
self._bounds = self._Reader.get_bounds(self._ch)
read_end = min(read_end, self._bounds[1])
else:
if read_end > self._end_sample:
read_end = self._end_sample
if read_start > read_end:
raise EOFError
# read data
data_dict = self._Reader.read(read_start, read_end, self._ch)
# handled all samples through read_end regardless of whether
# they were written to the output vector
self._read_start_sample = read_end + 1
# early escape for no data
if not data_dict:
if self._gapless:
# output empty samples if no data and gapless output
stop_index = next_index + read_end + 1 - read_start
out[next_index:stop_index] = self._fillvalue
next_index = stop_index
else:
# clear any existing tags
self._tag_queue.clear()
# add tag at next potential sample to indicate skip
self._queue_tags(self._read_start_sample, {})
continue
# read corresponding metadata
if self._DMDReader is not None:
meta_dict = self._DMDReader.read(read_start, read_end)
for sample, meta in meta_dict.items():
# add tags from Digital Metadata
# (in addition to default time and rate tags)
# eliminate sample_rate_* tags with duplicate info
meta.pop("sample_rate_denominator", None)
meta.pop("sample_rate_numerator", None)
# get center frequencies for rx_freq tag, squeeze()[()]
# to get single value if possible else pass as an array
cf = meta.pop("center_frequencies", None)
if cf is not None:
cf = cf.ravel().squeeze()[()]
tags = dict(
rx_freq=cf,
# all other metadata goes in metadata tag
metadata=meta,
)
self._queue_tags(sample, tags)
# add data and tags to output
next_continuous_sample = read_start
for sample, data in data_dict.items():
# detect data skip
if sample > next_continuous_sample:
if self._gapless:
# advance output by skipped number of samples
nskipped = sample - next_continuous_sample
sample_index = next_index + nskipped
out[next_index:sample_index] = self._fillvalue
next_index = sample_index
else:
# emit new time tag at sample to indicate skip
self._queue_tags(sample, {})
# output data
n = data.shape[0]
stop_index = next_index + n
end_sample = sample + n
out_dest = out[next_index:stop_index]
data_arr = data.squeeze()
out_dest[:] = data_arr
# overwrite missing values with fill values
missing_val_idx = self._ismissing(data_arr)
out_dest[missing_val_idx] = self._fillvalue
# output tags
for tag_sample in sorted(self._tag_queue.keys()):
if tag_sample < sample:
# drop tags from before current data block
del self._tag_queue[tag_sample]
continue
elif tag_sample >= end_sample:
# wait to output tags from after current data block
break
offset = (
self.nitems_written(0) # offset @ start of work
+ next_index # additional offset of data block
+ (tag_sample - sample))
tag_dict = self._tag_queue.pop(tag_sample)
for name, val in tag_dict.items():
self.add_item_tag(0, offset, pmt.intern(name), val,
self._id)
# advance next output index and continuous sample
next_index = stop_index # <=== next_index += n
next_continuous_sample = end_sample
except EOFError:
if self._repeat:
if self._start_sample is None:
self._read_start_sample = self._bounds[0]
else:
self._read_start_sample = self._start_sample
self._queue_tags(self._read_start_sample, {})
continue
else:
break
if next_index == 0:
# return WORK_DONE
return -1
return next_index
def get_gapless(self):
return self._gapless
def set_gapless(self, gapless):
self._gapless = gapless
def get_repeat(self):
return self._repeat
def set_repeat(self, repeat):
self._repeat = repeat
def __init__(
self,
top_level_dir,
channels=None,
start=None,
end=None,
repeat=False,
throttle=False,
gapless=False,
min_chunksize=None,
):
"""Read data from a directory containing Digital RF channels.
In addition to outputting samples from Digital RF format data, this
block also emits a 'properties' message containing inherent channel
properties and adds stream tags using the channel's accompanying
Digital Metadata. See the Notes section for details on what the
messages and stream tags contain.
Parameters
----------
top_level_dir : string
Either a single top-level directory containing Digital RF channel
directories, or a list of such. A directory can be a file system
path or a url, where the url points to a top level directory. Each
must be a local path, or start with 'http://'', 'file://'', or
'ftp://''.
Other Parameters
----------------
channels : None | string | int | iterable of previous, optional
If None, use all available channels in alphabetical order.
Otherwise, use the channels in the order specified in the given
iterable (a string or int is taken as a single-element iterable).
A string is used to specify the channel name, while an int is used
to specify the channel index in the sorted list of available
channel names.
start : None | string | int | iterable of previous, optional
Can be a single value or an iterable of values corresponding to
`channels` giving the start of the channel's playback.
If None or '', the start of the channel's available data is used.
If an integer, it is interpreted as a sample index given in the
number of samples since the epoch (time_since_epoch*sample_rate).
If a float, it is interpreted as a UTC timestamp (seconds since
epoch).
If a string, four forms are permitted:
1) a string which can be evaluated to an integer/float and
interpreted as above,
2) a string beginning with '+' and followed by an integer
(float) expression, interpreted as samples (seconds) from
the start of the data, and
3) a time in ISO8601 format, e.g. '2016-01-01T16:24:00Z'
4) 'now' ('nowish'), indicating the current time (rounded up)
end : None | string | int | iterable of previous, optional
Can be a single value or an iterable of values corresponding to
`channels` giving the end of the channel's playback.
If None or '', the end of the channel's available data is used.
See `start` for a description of how this value is interpreted.
repeat : bool, optional
If True, loop the data continuously from the start after the end
is reached. If False, stop after the data is read once.
throttle : bool, optional
If True, playback the samples at their recorded sample rate. If
False, read samples as quickly as possible.
gapless : bool, optional
If True, output zeroed samples for any missing data between start
and end. If False, skip missing samples and add an `rx_time` stream
tag to indicate the gap.
min_chunksize : None | int, optional
Minimum number of samples to output at once. This value can be used
to adjust the source's performance to reduce underruns and
processing time. If None, a sensible default will be used.
Notes
-----
A top-level directory must contain files in the format:
[channel]/[YYYY-MM-DDTHH-MM-SS]/rf@[seconds].[%03i milliseconds].h5
If more than one top level directory contains the same channel_name
subdirectory, this is considered the same channel. An error is raised
if their sample rates differ, or if their time periods overlap.
Upon start, this block sends 'properties' messages on its output
message port that contains a dictionaries with one key, the channel's
name, and a value which is a dictionary of properties found in the
channel's 'drf_properties.h5' file.
This block emits the following stream tags at the appropriate sample
for each of the channel's accompanying Digital Metadata samples:
rx_time : (int secs, float frac) tuple
Time since epoch of the sample.
rx_rate : float
Sample rate in Hz.
rx_freq : float | 1-D array of floats
Center frequency or frequencies of the subchannels based on
the 'center_frequencies' metadata field.
metadata : dict
Any additional Digital Metadata fields are added to this
dictionary tag of metadata.
"""
options = locals()
del options["self"]
del options["top_level_dir"]
del options["channels"]
del options["start"]
del options["end"]
del options["throttle"]
Reader = DigitalRFReader(top_level_dir)
available_channel_names = Reader.get_channels()
self._channel_names = self._get_channel_names(channels,
available_channel_names)
if start is None or isinstance(start, six.string_types):
start = [start] * len(self._channel_names)
try:
s_iter = iter(start)
except TypeError:
s_iter = iter([start])
if end is None or isinstance(end, six.string_types):
end = [end] * len(self._channel_names)
try:
e_iter = iter(end)
except TypeError:
e_iter = iter([end])
# make sources for each channel
self._channels = []
for ch, s, e in zip(self._channel_names, s_iter, e_iter):
chsrc = digital_rf_channel_source(os.path.join(top_level_dir, ch),
start=s,
end=e,
**options)
self._channels.append(chsrc)
out_sig_dtypes = [list(src.out_sig())[0] for src in self._channels]
out_sig = gr.io_signaturev(
len(out_sig_dtypes),
len(out_sig_dtypes),
[s.itemsize for s in out_sig_dtypes],
)
in_sig = gr.io_signature(0, 0, 0)
gr.hier_block2.__init__(
self,
name="digital_rf_source",
input_signature=in_sig,
output_signature=out_sig,
)
msg_port_name = pmt.intern("properties")
self.message_port_register_hier_out("properties")
for k, src in enumerate(self._channels):
if throttle:
throt = gnuradio.blocks.throttle(
list(src.out_sig())[0].itemsize,
float(src._sample_rate),
ignore_tags=True,
)
self.connect(src, throt, (self, k))
else:
self.connect(src, (self, k))
self.msg_connect(src, msg_port_name, self, msg_port_name)
def __init__(
self, channel_dir, start=None, end=None, repeat=False,
):
"""Initialize source to directory containing Digital RF channels.
Parameters
----------
channel_dir : string
Either a single channel directory containing 'drf_properties.h5'
and timestamped subdirectories with Digital RF files, or a list of
such. A directory can be a file system path or a url, where the url
points to a channel directory. Each must be a local path, or start
with 'http://'', 'file://'', or 'ftp://''.
Other Parameters
----------------
start : None | int/long | float | string
A value giving the start of the channel's playback.
If None or '', the start of the channel's available data is used.
If an integer, it is interpreted as a sample index given in the
number of samples since the epoch (time_since_epoch*sample_rate).
If a float, it is interpreted as a timestamp (seconds since epoch).
If a string, three forms are permitted:
1) a string which can be evaluated to an integer/float and
interpreted as above,
2) a string beginning with '+' and followed by an integer
(float) expression, interpreted as samples (seconds) from
the start of the data, and
3) a time in ISO8601 format, e.g. '2016-01-01T16:24:00Z'
end : None | int/long | float | string
A value giving the end of the channel's playback.
If None or '', the end of the channel's available data is used.
See `start` for a description of how this value is interpreted.
repeat : bool
If True, loop the data continuously from the start after the end
is reached. If False, stop after the data is read once.
Notes
-----
A channel directory must contain subdirectories/files in the format:
<YYYY-MM-DDTHH-MM-SS/rf@<seconds>.<%03i milliseconds>.h5
Each directory provided is considered the same channel. An error is
raised if their sample rates differ, or if their time periods overlap.
"""
if isinstance(channel_dir, six.string_types):
channel_dir = [channel_dir]
# eventually, we should re-factor DigitalRFReader and associated so
# that reading from a list of channel directories is possible
# with a DigitalRFChannelReader class or similar
# until then, split the path and use existing DigitalRFReader
top_level_dirs = []
chs = set()
for ch_dir in channel_dir:
top_level_dir, ch = os.path.split(ch_dir)
top_level_dirs.append(top_level_dir)
chs.add(ch)
if len(chs) == 1:
ch = chs.pop()
else:
raise ValueError('Channel directories must have the same name.')
self._ch = ch
self._Reader = DigitalRFReader(top_level_dirs)
self._properties = self._Reader.get_properties(self._ch)
typeclass = self._properties['H5Tget_class']
itemsize = self._properties['H5Tget_size']
is_complex = self._properties['is_complex']
vlen = self._properties['num_subchannels']
sr = self._properties['samples_per_second']
self._itemsize = itemsize
self._sample_rate = sr
self._sample_rate_pmt = pmt.from_double(float(sr))
# determine output signature from HDF5 type metadata
typedict = get_h5type(typeclass, itemsize, is_complex)
self._outtype = typedict['name']
self._itemtype = typedict['dtype']
if vlen == 1:
out_sig = [self._itemtype]
else:
out_sig = [(self._itemtype, vlen)]
gr.sync_block.__init__(
self,
name="digital_rf_channel_source",
in_sig=None,
out_sig=out_sig,
)
self.message_port_register_out(pmt.intern('metadata'))
self._id = pmt.intern(self._ch)
self._tag_queue = {}
self._start = start
self._end = end
self._repeat = repeat
try:
self._DMDReader = self._Reader.get_digital_metadata(self._ch)
except IOError:
self._DMDReader = None
# FIXME: should not be necessary, sets a large output buffer so that
# we don't underrun on frequent calls to work
self.set_output_multiple(int(sr))
class digital_rf_channel_source(gr.sync_block):
"""
docstring for block digital_rf_channel_source
"""
def __init__(
self, channel_dir, start=None, end=None, repeat=False,
):
"""Initialize source to directory containing Digital RF channels.
Parameters
----------
channel_dir : string
Either a single channel directory containing 'drf_properties.h5'
and timestamped subdirectories with Digital RF files, or a list of
such. A directory can be a file system path or a url, where the url
points to a channel directory. Each must be a local path, or start
with 'http://'', 'file://'', or 'ftp://''.
Other Parameters
----------------
start : None | int/long | float | string
A value giving the start of the channel's playback.
If None or '', the start of the channel's available data is used.
If an integer, it is interpreted as a sample index given in the
number of samples since the epoch (time_since_epoch*sample_rate).
If a float, it is interpreted as a timestamp (seconds since epoch).
If a string, three forms are permitted:
1) a string which can be evaluated to an integer/float and
interpreted as above,
2) a string beginning with '+' and followed by an integer
(float) expression, interpreted as samples (seconds) from
the start of the data, and
3) a time in ISO8601 format, e.g. '2016-01-01T16:24:00Z'
end : None | int/long | float | string
A value giving the end of the channel's playback.
If None or '', the end of the channel's available data is used.
See `start` for a description of how this value is interpreted.
repeat : bool
If True, loop the data continuously from the start after the end
is reached. If False, stop after the data is read once.
Notes
-----
A channel directory must contain subdirectories/files in the format:
<YYYY-MM-DDTHH-MM-SS/rf@<seconds>.<%03i milliseconds>.h5
Each directory provided is considered the same channel. An error is
raised if their sample rates differ, or if their time periods overlap.
"""
if isinstance(channel_dir, six.string_types):
channel_dir = [channel_dir]
# eventually, we should re-factor DigitalRFReader and associated so
# that reading from a list of channel directories is possible
# with a DigitalRFChannelReader class or similar
# until then, split the path and use existing DigitalRFReader
top_level_dirs = []
chs = set()
for ch_dir in channel_dir:
top_level_dir, ch = os.path.split(ch_dir)
top_level_dirs.append(top_level_dir)
chs.add(ch)
if len(chs) == 1:
ch = chs.pop()
else:
raise ValueError('Channel directories must have the same name.')
self._ch = ch
self._Reader = DigitalRFReader(top_level_dirs)
self._properties = self._Reader.get_properties(self._ch)
typeclass = self._properties['H5Tget_class']
itemsize = self._properties['H5Tget_size']
is_complex = self._properties['is_complex']
vlen = self._properties['num_subchannels']
sr = self._properties['samples_per_second']
self._itemsize = itemsize
self._sample_rate = sr
self._sample_rate_pmt = pmt.from_double(float(sr))
# determine output signature from HDF5 type metadata
typedict = get_h5type(typeclass, itemsize, is_complex)
self._outtype = typedict['name']
self._itemtype = typedict['dtype']
if vlen == 1:
out_sig = [self._itemtype]
else:
out_sig = [(self._itemtype, vlen)]
gr.sync_block.__init__(
self,
name="digital_rf_channel_source",
in_sig=None,
out_sig=out_sig,
)
self.message_port_register_out(pmt.intern('metadata'))
self._id = pmt.intern(self._ch)
self._tag_queue = {}
self._start = start
self._end = end
self._repeat = repeat
try:
self._DMDReader = self._Reader.get_digital_metadata(self._ch)
except IOError:
self._DMDReader = None
# FIXME: should not be necessary, sets a large output buffer so that
# we don't underrun on frequent calls to work
self.set_output_multiple(int(sr))
@staticmethod
def _parse_sample_identifier(iden, sample_rate=None, ref_index=None):
"""Get a sample index from different forms of identifiers.
Parameters
----------
iden : None | int/long | float | string
If None or '', None is returned to indicate that the index should
be automatically determined.
If an integer, it is returned as the sample index.
If a float, it is interpreted as a timestamp (seconds since epoch)
and the corresponding sample index is returned.
If a string, three forms are permitted:
1) a string which can be evaluated to an integer/float and
interpreted as above,
2) a string beginning with '+' and followed by an integer
(float) expression, interpreted as samples (seconds) from
`ref_index`, and
3) a time in ISO8601 format, e.g. '2016-01-01T16:24:00Z'
sample_rate : numpy.longdouble, required for float and time `iden`
Sample rate in Hz used to convert a time to a sample index.
ref_index : int/long, required for '+' string form of `iden`
Reference index from which string `iden` beginning with '+' are
offset.
Returns
-------
sample_index : long | None
Index to the identified sample given in the number of samples since
the epoch (time_since_epoch*sample_rate).
"""
is_relative = False
if iden is None or iden == '':
return None
elif isinstance(iden, six.string_types):
if iden.startswith('+'):
is_relative = True
iden = iden.lstrip('+')
try:
# int/long or float
iden = ast.literal_eval(iden)
except (ValueError, SyntaxError):
# convert datetime to float
dt = dateutil.parser.parse(iden)
epoch = datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)
iden = (dt - epoch).total_seconds()
if isinstance(iden, float):
if sample_rate is None:
raise ValueError(
'sample_rate required when time identifier is used.'
)
idx = long(np.uint64(iden*sample_rate))
else:
idx = long(iden)
if is_relative:
if ref_index is None:
raise ValueError(
'ref_index required when relative "+" identifier is used.'
)
return idx + ref_index
else:
return idx
def _queue_tags(self, sample, tags):
"""Queue stream tags to be attached to data in the work function.
In addition to the tags specified in the `tags` dictionary, this will
add `rx_time` and `rx_rate` tags giving the sample time and rate.
Parameters
----------
sample : int | long
Sample index for the sample to tag, given in the number of samples
since the epoch (time_since_epoch*sample_rate).
tags : dict
Dictionary containing the tags to add with keys specifying the tag
name. The value is cast as an appropriate pmt type, while the name
will be turned into a pmt string in the work function.
"""
# add to current queued tags for sample if applicable
tag_dict = self._tag_queue.get(sample, {})
if not tag_dict:
# add time and rate tags
time = sample/self._sample_rate
tag_dict['rx_time'] = pmt.make_tuple(
pmt.from_uint64(long(np.uint64(time))),
pmt.from_double(float(time % 1)),
)
tag_dict['rx_rate'] = self._sample_rate_pmt
for k, v in tags.items():
tag_dict[k] = pmt.to_pmt(v)
self._tag_queue[sample] = tag_dict
def start(self):
self._bounds = self._Reader.get_bounds(self._ch)
self._start_sample = self._parse_sample_identifier(
self._start, self._sample_rate, self._bounds[0],
)
self._end_sample = self._parse_sample_identifier(
self._end, self._sample_rate, self._bounds[0],
)
if self._start_sample is None:
self._global_index = self._bounds[0]
else:
self._global_index = self._start_sample
# add default tags to first sample
self._queue_tags(self._global_index, {})
# replace longdouble samples_per_second with float for pmt conversion
message_metadata = self._properties.copy()
message_metadata['samples_per_second'] = \
float(message_metadata['samples_per_second'])
self.message_port_pub(
pmt.intern('metadata'), pmt.to_pmt({self._ch: message_metadata}),
)
return super(digital_rf_channel_source, self).start()
def work(self, input_items, output_items):
out = output_items[0]
nsamples = len(out)
nout = 0
# repeat reading until we succeed or return
while nout < nsamples:
start_sample = self._global_index
# end_sample is inclusive, hence the -1
end_sample = self._global_index + (nsamples - nout) - 1
# creating a read function that has an output argument so data can
# be copied directly would be nice
# also should move EOFError checking into reader once watchdog
# bounds functionality is implemented
try:
if self._end_sample is None:
if end_sample > self._bounds[1]:
self._bounds = self._Reader.get_bounds(self._ch)
end_sample = min(end_sample, self._bounds[1])
else:
if end_sample > self._end_sample:
end_sample = self._end_sample
if start_sample > end_sample:
raise EOFError
data_dict = self._Reader.read(
start_sample, end_sample, self._ch,
)
for sample, data in data_dict.items():
# index into out starts at number of previously read
# samples plus the offset from what we requested
ks = nout + (sample - start_sample)
ke = ks + data.shape[0]
# out is zeroed, so only have to write samples we have
out[ks:ke] = data.squeeze()
# now read corresponding metadata
if self._DMDReader is not None:
meta_dict = self._DMDReader.read(
start_sample, end_sample,
)
for sample, meta in meta_dict.items():
# add center frequency tag from metadata
# (in addition to default time and rate tags)
tags = dict(
rx_freq=meta['center_frequencies'].ravel()[0]
)
self._queue_tags(sample, tags)
# add queued tags to stream
for sample, tag_dict in self._tag_queue.items():
offset = (
self.nitems_written(0) + nout +
(sample - start_sample)
)
for name, val in tag_dict.items():
self.add_item_tag(
0, offset, pmt.intern(name), val, self._id,
)
self._tag_queue.clear()
# no errors, so we read all the samples we wanted
# (end_sample is inclusive, hence the +1)
nread = (end_sample + 1 - start_sample)
nout += nread
self._global_index += nread
except EOFError:
if self._repeat:
if self._start_sample is None:
self._global_index = self._bounds[0]
else:
self._global_index = self._start_sample
self._queue_tags(self._global_index, {})
continue
else:
break
if nout == 0:
# return WORK_DONE
return -1
return nout
def __init__(
self, top_level_dir, channels=None, start=None, end=None,
repeat=False, throttle=False,
):
"""Initialize source to directory containing Digital RF channels.
Parameters
----------
top_level_dir : string
Either a single top level directory containing Digital RF channel
directories, or a list of such. A directory can be a file system
path or a url, where the url points to a top level directory. Each
must be a local path, or start with 'http://'', 'file://'', or
'ftp://''.
Other Parameters
----------------
channels : None | string | int | iterable of previous
If None, use all available channels in alphabetical order.
Otherwise, use the channels in the order specified in the given
iterable (a string or int is taken as a single-element iterable).
A string is used to specify the channel name, while an int is used
to specify the channel index in the sorted list of available
channel names.
start : None | string | long | iterable of previous
Can be a single value or an iterable of values corresponding to
`channels` giving the start of the channel's playback.
If None or '', the start of the channel's available data is used.
If an integer, it is interpreted as a sample index given in the
number of samples since the epoch (time_since_epoch*sample_rate).
If a float, it is interpreted as a timestamp (seconds since epoch).
If a string, three forms are permitted:
1) a string which can be evaluated to an integer/float and
interpreted as above,
2) a string beginning with '+' and followed by an integer
(float) expression, interpreted as samples (seconds) from
the start of the data, and
3) a time in ISO8601 format, e.g. '2016-01-01T16:24:00Z'
end : None | string | long | iterable of previous
Can be a single value or an iterable of values corresponding to
`channels` giving the end of the channel's playback.
If None or '', the end of the channel's available data is used.
See `start` for a description of how this value is interpreted.
repeat : bool
If True, loop the data continuously from the start after the end
is reached. If False, stop after the data is read once.
throttle : bool
If True, playback the samples at their recorded sample rate. If
False, read samples as quickly as possible.
Notes
-----
A top level directory must contain files in the format:
<channel>/<YYYY-MM-DDTHH-MM-SS/rf@<seconds>.<%03i milliseconds>.h5
If more than one top level directory contains the same channel_name
subdirectory, this is considered the same channel. An error is raised
if their sample rates differ, or if their time periods overlap.
"""
Reader = DigitalRFReader(top_level_dir)
available_channel_names = Reader.get_channels()
self._channel_names = self._get_channel_names(
channels, available_channel_names,
)
if start is None:
start = [None]*len(self._channel_names)
try:
s_iter = iter(start)
except TypeError:
s_iter = iter([start])
if end is None:
end = [None]*len(self._channel_names)
try:
e_iter = iter(end)
except TypeError:
e_iter = iter([end])
# make sources for each channel
self._channels = []
for ch, s, e in zip(self._channel_names, s_iter, e_iter):
chsrc = digital_rf_channel_source(
os.path.join(top_level_dir, ch), start=s, end=e, repeat=repeat,
)
self._channels.append(chsrc)
out_sig_dtypes = [src.out_sig()[0] for src in self._channels]
out_sig = gr.io_signaturev(
len(out_sig_dtypes), len(out_sig_dtypes),
[s.itemsize for s in out_sig_dtypes],
)
in_sig = gr.io_signature(0, 0, 0)
gr.hier_block2.__init__(
self,
name="digital_rf_source",
input_signature=in_sig,
output_signature=out_sig,
)
msg_port_name = pmt.intern('metadata')
self.message_port_register_hier_out('metadata')
for k, src in enumerate(self._channels):
if throttle:
throt = gnuradio.blocks.throttle(
src.out_sig()[0].itemsize, src._sample_rate,
ignore_tags=True,
)
self.connect(src, throt, (self, k))
else:
self.connect(src, (self, k))
self.msg_connect(src, msg_port_name, self, msg_port_name)