def getImgByTimeStamp(self, timestamp, update_index=False):
if timestamp not in self._timestamps:
raise RuntimeError("Timestamp not found")
if update_index:
self._offset = self._timestamps.index(timestamp)
relpath = _ts_date_to_path(self.name, self.extension, timestamp, 0)
img_path = path.join(self.path, relpath)
if self._err_on_access:
if ts_format_date(timestamp) in self._ignore_ts:
raise PCExSkippedImage(timestamp)
if ts_format_date(timestamp) in self._existing_ts:
raise PCExExistingImage(timestamp)
if not path.exists(img_path):
raise PCExMissingImage(timestamp, img_path)
img = self.load_pickled_image(timestamp)
if img is None:
img = TimeStreamImage(dt=timestamp)
img.parent_timestream = self
img.path = img_path
try:
img_date = ts_format_date(img.datetime)
img.data = self.image_data[img_date]
except KeyError:
img.data = {}
return img
def iter_by_files(self, ignored_timestamps=[]):
for fpath in all_files_with_ext(
self.path, self.extension, cs=False):
img = TimeStreamImage()
img.parent_timestream = self
img.path = fpath
img_date = ts_format_date(img.datetime)
# skip images in ignored_timestamps
if img_date in ignored_timestamps:
LOG.info("Skip processing data at {}".format(img.datetime))
continue
try:
img.data = self.image_data[img_date]
except KeyError:
img.data = {}
yield img
def curr(self):
time = self._timestamps[self._offset]
relpath = _ts_date_to_path(self.name, self.extension, time, 0)
img_path = path.join(self.path, relpath)
img = self.load_pickled_image(time)
if img is None:
img = TimeStreamImage(dt=time)
img.parent_timestream = self
img.path = img_path
try:
img_date = ts_format_date(img.datetime)
img.data = self.image_data[img_date]
except KeyError:
img.data = {}
return img
def test_timestream_write(self):
ts = TimeStream()
ts.version = 1
ts.create(self.tmp_path, ext="jpg")
self.assertEqual(ts.path, self.tmp_path)
self.assertDictEqual(ts.image_data, {})
self.assertDictEqual(ts.data, {})
for _ in range(10):
img = TimeStreamImage()
arr = np.arange(300, dtype="uint8")
arr = arr.reshape((10, 10, 3))
date = dt.datetime.now()
str_date = ts_format_date(date)
img.pixels = arr
img.datetime = date
img.data["fake"] = True
ts.write_image(img)
self.assertIn(str_date, ts.image_data)
self.assertDictEqual(img.data, ts.image_data[str_date])
self.assertTrue(path.exists, img.path)
def __init__(self, ts_path=None, version=None, interval=None,
start=None, end=None, start_hour=None, end_hour=None,
ignore_ts=[], existing_ts=[], err_on_access=False):
"""Class to got back and forth on a TimeStream
Use This class when you need to traverse the timestream both forwards
and backwards in time.
Args:
version(int): Passed directly to TimeStream
interval(int): Interval between time stamps
start(datetime): Start of time stream
end(datetime): End of time stream
start_hour(datetime): Starting hour within every day of time stream
end_hour(datetime): Ending hour within every day of time stream
ignore_ts(list): Timestamps to be ignored.
existing_ts(list): existing timestamps that should not be recalculated.
err_on_access: If false we make all error checks silently in __init__.
If true we raise errors when accessing imgs.
Attributes:
_timestamps(list): List of strings that index all existing image files
in this timestream
_offset(int): Current offset within _timestamps.
"""
super(TimeStreamTraverser, self).__init__(version=version)
self.load(ts_path)
self._err_on_access = err_on_access
self._ignore_ts = ignore_ts
self._existing_ts = existing_ts
self._offset = 0
self._timestamps = []
# FIXME: Following is practically equal to
# TimeStream.iter_by_timepoints.
if not start or start < self.start_datetime:
start = self.start_datetime
if not end or end > self.end_datetime:
end = self.end_datetime
if not interval:
interval = self.interval
# fix hour range if given
if start_hour is not None:
start = dt.datetime.combine(start.date(), start_hour)
if end_hour is not None:
end = dt.datetime.combine(end.date(), end_hour)
# iterate threw times
for time in iter_date_range(start, end, interval):
# apply hour range if given
if start_hour is not None:
hrstart = dt.datetime.combine(time.date(), start_hour)
if time < hrstart:
continue
if end_hour is not None:
hrend = dt.datetime.combine(time.date(), end_hour)
if time > hrend:
continue
# skip images in ignore_ts
if ts_format_date(time) in self._ignore_ts:
if not self._err_on_access:
continue
# skip images in existing_ts
if ts_format_date(time) in self._existing_ts:
if not self._err_on_access:
continue
# Do not add if path dosn't exist
relpath = _ts_date_to_path(self.name, self.extension, time, 0)
img_path = path.join(self.path, relpath)
if not path.exists(img_path):
if not self._err_on_access:
continue
self._timestamps.append(time)
def iter_by_timepoints(self, remove_gaps=True, start=None, end=None,
interval=None, start_hour=None, end_hour=None,
ignored_timestamps=[]):
"""
Iterate over a TimeStream in chronological order, yielding a
TimeStreamImage instance for each timepoint. If ``remove_gaps`` is
False, yield None for missing images.
"""
if not start or start < self.start_datetime:
start = self.start_datetime
if not end or end > self.end_datetime:
end = self.end_datetime
if not interval:
interval = self.interval
# fix hour range if given
if start_hour is not None:
start = dt.datetime.combine(start.date(), start_hour)
if end_hour is not None:
end = dt.datetime.combine(end.date(), end_hour)
# iterate thru times
for time in iter_date_range(start, end, interval):
# skip images in ignored_timestamps
if ts_format_date(time) in ignored_timestamps:
LOG.info("Skip processing data at {}".format(time))
continue
# apply hour range if given
if start_hour is not None:
hrstart = dt.datetime.combine(time.date(), start_hour)
if time < hrstart:
continue
if end_hour is not None:
hrend = dt.datetime.combine(time.date(), end_hour)
if time > hrend:
continue
# Format the path below the ts root
relpath = _ts_date_to_path(self.name, self.extension, time, 0)
# Join to make "absolute" path, i.e. path including ts_path
img_path = path.join(self.path, relpath)
# not-so-silently fail if we can't find the image
if path.exists(img_path):
LOG.debug("Image at {} in {} is {}.".format(time, self.path,
img_path))
else:
LOG.debug("Expected image {} at {} did not exist.".format(
img_path, time, self.path))
img_path = None
if remove_gaps and img_path is None:
continue
elif img_path is None:
img = TimeStreamImage(dt=time)
img.pixels = np.array([])
yield img
else:
img = self.load_pickled_image(time)
if img is None:
img = TimeStreamImage(dt=time)
img.parent_timestream = self
img.path = img_path
try:
img_date = ts_format_date(img.datetime)
img.data = self.image_data[img_date]
except KeyError:
img.data = {}
yield img
def write_image(self, image, overwrite_mode="skip"):
if not self.name:
msg = "write_image() must be called on instance with valid name"
LOG.error(msg)
raise RuntimeError(msg)
if not self.path:
msg = "write_image() must be called on instance with valid path"
LOG.error(msg)
raise RuntimeError(msg)
if not isinstance(image, TimeStreamImage):
msg = "write_image() must be given an instance of TimeStreamImage"
LOG.error(msg)
raise TypeError(msg)
if not isinstance(overwrite_mode, str):
msg = "overwrite_mode must be a str"
LOG.error(msg)
raise TypeError(msg)
if overwrite_mode not in {"skip", "increment", "overwrite", "raise"}:
msg = "Invalid overwrite_mode {}.".format(overwrite_mode)
LOG.error(msg)
raise ValueError(msg)
if image.pixels is None:
msg = "Image must have pixels to be able to be written"
LOG.error(msg)
raise ValueError(msg)
if self.version == 1:
fpath = _ts_date_to_path(self.name, self.extension,
image.datetime, 0)
fpath = path.join(self.path, fpath)
if path.exists(fpath):
if overwrite_mode == "skip":
return
elif overwrite_mode == "increment":
subsec = 0
while path.exists(fpath) and subsec < 100:
subsec += 1
fpath = _ts_date_to_path(self.name, image.datetime,
subsec)
if path.exists(fpath):
msg = "Too many images at timepoint {}".format(
ts_format_date(image.datetime))
LOG.error(msg)
raise ValueError(msg)
elif overwrite_mode == "overwrite":
# We don't do anything here if we want to overwrite
pass
elif overwrite_mode == "raise":
msg = "Image already exists at {}".format(fpath)
LOG.error(msg)
raise ValueError(msg)
# Update timestream if required
if image.datetime > self.end_datetime:
self.end_datetime = image.datetime
if image.datetime < self.start_datetime:
self.start_datetime = image.datetime
self.image_data[ts_format_date(image.datetime)] = image.data
self.write_metadata()
# FIXME: pass the overwrite_mode
image.write(fpath=fpath, overwrite=True)
self.write_pickled_image(image, overwrite=True)
else:
raise NotImplementedError("v2 timestreams not implemented yet")
def __init__(self, ts_path=None, version=None, interval=None,
start=None, end=None, start_hour=None, end_hour=None,
ignored_timestamps=[]):
"""Class to got back and forth on a TimeStream
Use This class when you need to traverse the timestream both forwards
and backwards in time.
Args:
version(int): Passed directly to TimeStream
interval(int): Interval between time stamps
start(datetime): Start of time stream
end(datetime): End of time stream
start_hour(datetime): Starting hour within every day of time stream
end_hour(datetime): Ending hour within every day of time stream
ignored_timestamps(list): List of ignore time stamps.0
Attributes:
_timestamps(list): List of strings that index all existing image files
in this timestream
_offset(int): Current offset within _timestamps.
"""
super(TimeStreamTraverser, self).__init__(version=version)
self.load(ts_path)
self._offset = 0
self._timestamps = []
# FIXME: Following is practically equal to
# TimeStream.iter_by_timepoints.
if not start or start < self.start_datetime:
start = self.start_datetime
if not end or end > self.end_datetime:
end = self.end_datetime
if not interval:
interval = self.interval
# fix hour range if given
if start_hour is not None:
start = dt.datetime.combine(start.date(), start_hour)
if end_hour is not None:
end = dt.datetime.combine(end.date(), end_hour)
# iterate thru times
for time in iter_date_range(start, end, interval):
# skip images in ignored_timestamps
if ts_format_date(time) in ignored_timestamps:
continue
# apply hour range if given
if start_hour is not None:
hrstart = dt.datetime.combine(time.date(), start_hour)
if time < hrstart:
continue
if end_hour is not None:
hrend = dt.datetime.combine(time.date(), end_hour)
if time > hrend:
continue
# If path exists add index to _timestamps
relpath = _ts_date_to_path(self.name, self.extension, time, 0)
img_path = path.join(self.path, relpath)
if path.exists(img_path):
self._timestamps.append(time)