def fetch_raw_data(self):
"""
Fetch the source data for this request
:return:
"""
for stream_key, stream_parameters in self.stream_parameters.iteritems():
other_streams = set(self.stream_parameters)
other_streams.remove(stream_key)
should_pad = stream_key != self.stream_key
if not stream_key.is_virtual:
log.debug('<%s> Fetching raw data for %s', self.request_id, stream_key.as_refdes())
sd = StreamDataset(stream_key, self.coefficients, self.uflags, other_streams, self.request_id)
try:
sd.fetch_raw_data(self.time_range, self.limit, should_pad)
self.datasets[stream_key] = sd
except MissingDataException as e:
if stream_key == self.stream_key:
raise MissingDataException("Query returned no results for primary stream")
elif stream_key in self.stream_key.stream.source_streams:
raise MissingDataException("Query returned no results for source stream")
else:
log.error('<%s> %s', self.request_id, e.message)
else:
log.debug('<%s> Creating empty dataset for virtual stream: %s',
self.request_id, stream_key.as_refdes())
sd = StreamDataset(stream_key, self.coefficients, self.uflags, other_streams, self.request_id)
self.datasets[stream_key] = sd
def fetch_raw_data(self):
"""
Fetch the source data for this request
:return:
"""
for stream_key, stream_parameters in self.stream_parameters.iteritems(
):
other_streams = set(self.stream_parameters)
other_streams.remove(stream_key)
should_pad = stream_key != self.stream_key
if not stream_key.is_virtual:
log.debug('<%s> Fetching raw data for %s', self.request_id,
stream_key.as_refdes())
sd = StreamDataset(stream_key, self.coefficients, self.uflags,
other_streams, self.request_id)
try:
sd.fetch_raw_data(self.time_range, self.limit, should_pad)
self.datasets[stream_key] = sd
except MissingDataException as e:
if stream_key == self.stream_key:
raise MissingDataException(
"Query returned no results for primary stream")
elif stream_key in self.stream_key.stream.source_streams:
raise MissingDataException(
"Query returned no results for source stream")
else:
log.error('<%s> %s', self.request_id, e.message)
else:
log.debug('<%s> Creating empty dataset for virtual stream: %s',
self.request_id, stream_key.as_refdes())
sd = StreamDataset(stream_key, self.coefficients, self.uflags,
other_streams, self.request_id)
self.datasets[stream_key] = sd
def fetch_raw_data(self):
"""
Fetch the source data for this request
:return:
"""
# Start fetching calibration data from Asset Management
am_events = {}
am_futures = {}
for stream_key in self.stream_parameters:
refdes = '-'.join(
(stream_key.subsite, stream_key.node, stream_key.sensor))
am_futures[stream_key] = self.asset_management.get_events_async(
refdes)
# Resolve calibration data futures and attach to instrument data
for stream_key in am_futures:
events = am_futures[stream_key].result()
am_events[stream_key] = events
# Start fetching instrument data
for stream_key, stream_parameters in self.stream_parameters.iteritems(
):
other_streams = set(self.stream_parameters)
other_streams.remove(stream_key)
should_pad = stream_key != self.stream_key
if not stream_key.is_virtual:
log.debug('<%s> Fetching raw data for %s', self.request_id,
stream_key.as_refdes())
sd = StreamDataset(stream_key, self.uflags, other_streams,
self.request_id)
sd.events = am_events[stream_key]
try:
sd.fetch_raw_data(self.time_range, self.limit, should_pad)
self.datasets[stream_key] = sd
except MissingDataException as e:
if stream_key == self.stream_key:
raise MissingDataException(
"Query returned no results for primary stream")
elif stream_key.stream in self.stream_key.stream.source_streams:
raise MissingDataException(
"Query returned no results for source stream")
else:
log.error('<%s> %s', self.request_id, e.message)
else:
log.debug('<%s> Creating empty dataset for virtual stream: %s',
self.request_id, stream_key.as_refdes())
sd = StreamDataset(stream_key, self.uflags, other_streams,
self.request_id)
sd.events = am_events[stream_key]
self.datasets[stream_key] = sd
self._exclude_flagged_data()
self._exclude_nondeployed_data()
# Verify data still exists after masking virtual
message = 'Query returned no results for %s stream (due to deployment or annotation mask)'
if self.stream_key.is_virtual:
found_streams = [
stream.stream for stream in self.datasets
if self.datasets[stream]
]
if not any(stream in self.stream_key.stream.source_streams
for stream in found_streams):
raise MissingDataException(message % 'source')
# real
else:
primary_stream_dataset = self.datasets[self.stream_key]
if not primary_stream_dataset.datasets:
raise MissingDataException(message % 'primary')
# Remove any empty, non-virtual supporting datasets
for stream_key in list(self.datasets):
if not stream_key.is_virtual:
if not self.datasets[stream_key].datasets:
del self.datasets[stream_key]
def fetch_raw_data(self):
"""
Fetch the source data for this request
:return:
"""
# Start fetching calibration data from Asset Management
am_events = {}
am_futures = {}
for stream_key in self.stream_parameters:
refdes = '-'.join((stream_key.subsite, stream_key.node, stream_key.sensor))
am_futures[stream_key] = self.asset_management.get_events_async(refdes)
# Resolve calibration data futures and attach to instrument data
for stream_key in am_futures:
events = am_futures[stream_key].result()
am_events[stream_key] = events
# Start fetching instrument data
for stream_key, stream_parameters in self.stream_parameters.iteritems():
other_streams = set(self.stream_parameters)
other_streams.remove(stream_key)
should_pad = stream_key != self.stream_key
if not stream_key.is_virtual:
log.debug('<%s> Fetching raw data for %s', self.request_id, stream_key.as_refdes())
sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id)
sd.events = am_events[stream_key]
try:
sd.fetch_raw_data(self.time_range, self.limit, should_pad)
self.datasets[stream_key] = sd
except MissingDataException as e:
if stream_key == self.stream_key:
raise MissingDataException("Query returned no results for primary stream")
elif stream_key.stream in self.stream_key.stream.source_streams:
raise MissingDataException("Query returned no results for source stream")
else:
log.error('<%s> %s', self.request_id, e.message)
else:
log.debug('<%s> Creating empty dataset for virtual stream: %s',
self.request_id, stream_key.as_refdes())
sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id)
sd.events = am_events[stream_key]
self.datasets[stream_key] = sd
# Fetch annotations
self._insert_annotations()
self._exclude_flagged_data()
self._exclude_nondeployed_data()
# Verify data still exists after masking virtual
message = 'Query returned no results for %s stream (due to deployment or annotation mask)'
if self.stream_key.is_virtual:
found_streams = [stream.stream for stream in self.datasets
if self.datasets[stream]]
if not any(stream in self.stream_key.stream.source_streams for stream in found_streams):
raise MissingDataException(message % 'source')
# real
else:
primary_stream_dataset = self.datasets[self.stream_key]
if not primary_stream_dataset.datasets:
raise MissingDataException(message % 'primary')
# Remove any empty, non-virtual supporting datasets
for stream_key in list(self.datasets):
if not stream_key.is_virtual:
if not self.datasets[stream_key].datasets:
del self.datasets[stream_key]