def main(opts):
setup_module_logging(logging.ERROR)
ts_name = opts["-i"]
out_fh = open(opts['-o'], 'w')
ts_info = ts_get_manifest(ts_name)
times = setup_header(ts_info)
out_csv = csv.writer(out_fh)
header = ["Date", ]
header.extend(times)
out_csv.writerow(header)
ts = TimeStream()
ts.load(ts_name)
res_dict = {}
print("Collecting image data for {}...".format(ts_name))
count = 0
for img in ts.iter_by_timepoints():
if count % 5 == 0:
print("Processed {} images!".format(count), end='\r')
sys.stdout.flush()
count += 1
img_dt = img.datetime
try:
res_dict[img_dt.date()][img_dt.time().isoformat()] = 1
except KeyError:
res_dict[img_dt.date()] = {img_dt.time().isoformat(): 1}
print("Processed {} images!".format(count))
print("Done collecting image sums, now making the table")
for this_date, times in sorted(res_dict.items()):
row = []
row.append(this_date.isoformat())
start_today = datetime.combine(this_date.today(), time.min)
end_today = datetime.combine(this_date.today(), time.max)
all_times = iter_date_range(start_today, end_today,
ts_info['interval'] * 60)
for timepoint in all_times:
try:
row.append(times[timepoint.time().isoformat()])
except KeyError:
row.append("0")
out_csv.writerow(row)
print("All done!")
def __init__(self, ts_path=None, version=None, interval=None,
start=None, end=None, start_hour=None, end_hour=None,
ignore_ts=[], existing_ts=[], err_on_access=False):
"""Class to got back and forth on a TimeStream
Use This class when you need to traverse the timestream both forwards
and backwards in time.
Args:
version(int): Passed directly to TimeStream
interval(int): Interval between time stamps
start(datetime): Start of time stream
end(datetime): End of time stream
start_hour(datetime): Starting hour within every day of time stream
end_hour(datetime): Ending hour within every day of time stream
ignore_ts(list): Timestamps to be ignored.
existing_ts(list): existing timestamps that should not be recalculated.
err_on_access: If false we make all error checks silently in __init__.
If true we raise errors when accessing imgs.
Attributes:
_timestamps(list): List of strings that index all existing image files
in this timestream
_offset(int): Current offset within _timestamps.
"""
super(TimeStreamTraverser, self).__init__(version=version)
self.load(ts_path)
self._err_on_access = err_on_access
self._ignore_ts = ignore_ts
self._existing_ts = existing_ts
self._offset = 0
self._timestamps = []
# FIXME: Following is practically equal to
# TimeStream.iter_by_timepoints.
if not start or start < self.start_datetime:
start = self.start_datetime
if not end or end > self.end_datetime:
end = self.end_datetime
if not interval:
interval = self.interval
# fix hour range if given
if start_hour is not None:
start = dt.datetime.combine(start.date(), start_hour)
if end_hour is not None:
end = dt.datetime.combine(end.date(), end_hour)
# iterate threw times
for time in iter_date_range(start, end, interval):
# apply hour range if given
if start_hour is not None:
hrstart = dt.datetime.combine(time.date(), start_hour)
if time < hrstart:
continue
if end_hour is not None:
hrend = dt.datetime.combine(time.date(), end_hour)
if time > hrend:
continue
# skip images in ignore_ts
if ts_format_date(time) in self._ignore_ts:
if not self._err_on_access:
continue
# skip images in existing_ts
if ts_format_date(time) in self._existing_ts:
if not self._err_on_access:
continue
# Do not add if path dosn't exist
relpath = _ts_date_to_path(self.name, self.extension, time, 0)
img_path = path.join(self.path, relpath)
if not path.exists(img_path):
if not self._err_on_access:
continue
self._timestamps.append(time)
def iter_by_timepoints(self, remove_gaps=True, start=None, end=None,
interval=None, start_hour=None, end_hour=None,
ignored_timestamps=[]):
"""
Iterate over a TimeStream in chronological order, yielding a
TimeStreamImage instance for each timepoint. If ``remove_gaps`` is
False, yield None for missing images.
"""
if not start or start < self.start_datetime:
start = self.start_datetime
if not end or end > self.end_datetime:
end = self.end_datetime
if not interval:
interval = self.interval
# fix hour range if given
if start_hour is not None:
start = dt.datetime.combine(start.date(), start_hour)
if end_hour is not None:
end = dt.datetime.combine(end.date(), end_hour)
# iterate thru times
for time in iter_date_range(start, end, interval):
# skip images in ignored_timestamps
if ts_format_date(time) in ignored_timestamps:
LOG.info("Skip processing data at {}".format(time))
continue
# apply hour range if given
if start_hour is not None:
hrstart = dt.datetime.combine(time.date(), start_hour)
if time < hrstart:
continue
if end_hour is not None:
hrend = dt.datetime.combine(time.date(), end_hour)
if time > hrend:
continue
# Format the path below the ts root
relpath = _ts_date_to_path(self.name, self.extension, time, 0)
# Join to make "absolute" path, i.e. path including ts_path
img_path = path.join(self.path, relpath)
# not-so-silently fail if we can't find the image
if path.exists(img_path):
LOG.debug("Image at {} in {} is {}.".format(time, self.path,
img_path))
else:
LOG.debug("Expected image {} at {} did not exist.".format(
img_path, time, self.path))
img_path = None
if remove_gaps and img_path is None:
continue
elif img_path is None:
img = TimeStreamImage(dt=time)
img.pixels = np.array([])
yield img
else:
img = self.load_pickled_image(time)
if img is None:
img = TimeStreamImage(dt=time)
img.parent_timestream = self
img.path = img_path
try:
img_date = ts_format_date(img.datetime)
img.data = self.image_data[img_date]
except KeyError:
img.data = {}
yield img
def setup_header(ts_info):
start_today = datetime.combine(date.today(), time.min)
end_today = datetime.combine(date.today(), time.max)
times = iter_date_range(start_today, end_today, ts_info['interval'] * 60)
times = [x.strftime("%H:%M:%S") for x in times]
return times
def __init__(self, ts_path=None, version=None, interval=None,
start=None, end=None, start_hour=None, end_hour=None,
ignored_timestamps=[]):
"""Class to got back and forth on a TimeStream
Use This class when you need to traverse the timestream both forwards
and backwards in time.
Args:
version(int): Passed directly to TimeStream
interval(int): Interval between time stamps
start(datetime): Start of time stream
end(datetime): End of time stream
start_hour(datetime): Starting hour within every day of time stream
end_hour(datetime): Ending hour within every day of time stream
ignored_timestamps(list): List of ignore time stamps.0
Attributes:
_timestamps(list): List of strings that index all existing image files
in this timestream
_offset(int): Current offset within _timestamps.
"""
super(TimeStreamTraverser, self).__init__(version=version)
self.load(ts_path)
self._offset = 0
self._timestamps = []
# FIXME: Following is practically equal to
# TimeStream.iter_by_timepoints.
if not start or start < self.start_datetime:
start = self.start_datetime
if not end or end > self.end_datetime:
end = self.end_datetime
if not interval:
interval = self.interval
# fix hour range if given
if start_hour is not None:
start = dt.datetime.combine(start.date(), start_hour)
if end_hour is not None:
end = dt.datetime.combine(end.date(), end_hour)
# iterate thru times
for time in iter_date_range(start, end, interval):
# skip images in ignored_timestamps
if ts_format_date(time) in ignored_timestamps:
continue
# apply hour range if given
if start_hour is not None:
hrstart = dt.datetime.combine(time.date(), start_hour)
if time < hrstart:
continue
if end_hour is not None:
hrend = dt.datetime.combine(time.date(), end_hour)
if time > hrend:
continue
# If path exists add index to _timestamps
relpath = _ts_date_to_path(self.name, self.extension, time, 0)
img_path = path.join(self.path, relpath)
if path.exists(img_path):
self._timestamps.append(time)