def main(opts): setup_module_logging(logging.ERROR) ts_name = opts["-i"] out_fh = open(opts['-o'], 'w') ts_info = ts_get_manifest(ts_name) times = setup_header(ts_info) out_csv = csv.writer(out_fh) header = ["Date", ] header.extend(times) out_csv.writerow(header) ts = TimeStream() ts.load(ts_name) res_dict = {} print("Collecting image data for {}...".format(ts_name)) count = 0 for img in ts.iter_by_timepoints(): if count % 5 == 0: print("Processed {} images!".format(count), end='\r') sys.stdout.flush() count += 1 img_dt = img.datetime try: res_dict[img_dt.date()][img_dt.time().isoformat()] = 1 except KeyError: res_dict[img_dt.date()] = {img_dt.time().isoformat(): 1} print("Processed {} images!".format(count)) print("Done collecting image sums, now making the table") for this_date, times in sorted(res_dict.items()): row = [] row.append(this_date.isoformat()) start_today = datetime.combine(this_date.today(), time.min) end_today = datetime.combine(this_date.today(), time.max) all_times = iter_date_range(start_today, end_today, ts_info['interval'] * 60) for timepoint in all_times: try: row.append(times[timepoint.time().isoformat()]) except KeyError: row.append("0") out_csv.writerow(row) print("All done!")
def __init__(self, ts_path=None, version=None, interval=None, start=None, end=None, start_hour=None, end_hour=None, ignore_ts=[], existing_ts=[], err_on_access=False): """Class to got back and forth on a TimeStream Use This class when you need to traverse the timestream both forwards and backwards in time. Args: version(int): Passed directly to TimeStream interval(int): Interval between time stamps start(datetime): Start of time stream end(datetime): End of time stream start_hour(datetime): Starting hour within every day of time stream end_hour(datetime): Ending hour within every day of time stream ignore_ts(list): Timestamps to be ignored. existing_ts(list): existing timestamps that should not be recalculated. err_on_access: If false we make all error checks silently in __init__. If true we raise errors when accessing imgs. Attributes: _timestamps(list): List of strings that index all existing image files in this timestream _offset(int): Current offset within _timestamps. """ super(TimeStreamTraverser, self).__init__(version=version) self.load(ts_path) self._err_on_access = err_on_access self._ignore_ts = ignore_ts self._existing_ts = existing_ts self._offset = 0 self._timestamps = [] # FIXME: Following is practically equal to # TimeStream.iter_by_timepoints. if not start or start < self.start_datetime: start = self.start_datetime if not end or end > self.end_datetime: end = self.end_datetime if not interval: interval = self.interval # fix hour range if given if start_hour is not None: start = dt.datetime.combine(start.date(), start_hour) if end_hour is not None: end = dt.datetime.combine(end.date(), end_hour) # iterate threw times for time in iter_date_range(start, end, interval): # apply hour range if given if start_hour is not None: hrstart = dt.datetime.combine(time.date(), start_hour) if time < hrstart: continue if end_hour is not None: hrend = dt.datetime.combine(time.date(), end_hour) if time > hrend: continue # skip images in ignore_ts if ts_format_date(time) in self._ignore_ts: if not self._err_on_access: continue # skip images in existing_ts if ts_format_date(time) in self._existing_ts: if not self._err_on_access: continue # Do not add if path dosn't exist relpath = _ts_date_to_path(self.name, self.extension, time, 0) img_path = path.join(self.path, relpath) if not path.exists(img_path): if not self._err_on_access: continue self._timestamps.append(time)
def iter_by_timepoints(self, remove_gaps=True, start=None, end=None, interval=None, start_hour=None, end_hour=None, ignored_timestamps=[]): """ Iterate over a TimeStream in chronological order, yielding a TimeStreamImage instance for each timepoint. If ``remove_gaps`` is False, yield None for missing images. """ if not start or start < self.start_datetime: start = self.start_datetime if not end or end > self.end_datetime: end = self.end_datetime if not interval: interval = self.interval # fix hour range if given if start_hour is not None: start = dt.datetime.combine(start.date(), start_hour) if end_hour is not None: end = dt.datetime.combine(end.date(), end_hour) # iterate thru times for time in iter_date_range(start, end, interval): # skip images in ignored_timestamps if ts_format_date(time) in ignored_timestamps: LOG.info("Skip processing data at {}".format(time)) continue # apply hour range if given if start_hour is not None: hrstart = dt.datetime.combine(time.date(), start_hour) if time < hrstart: continue if end_hour is not None: hrend = dt.datetime.combine(time.date(), end_hour) if time > hrend: continue # Format the path below the ts root relpath = _ts_date_to_path(self.name, self.extension, time, 0) # Join to make "absolute" path, i.e. path including ts_path img_path = path.join(self.path, relpath) # not-so-silently fail if we can't find the image if path.exists(img_path): LOG.debug("Image at {} in {} is {}.".format(time, self.path, img_path)) else: LOG.debug("Expected image {} at {} did not exist.".format( img_path, time, self.path)) img_path = None if remove_gaps and img_path is None: continue elif img_path is None: img = TimeStreamImage(dt=time) img.pixels = np.array([]) yield img else: img = self.load_pickled_image(time) if img is None: img = TimeStreamImage(dt=time) img.parent_timestream = self img.path = img_path try: img_date = ts_format_date(img.datetime) img.data = self.image_data[img_date] except KeyError: img.data = {} yield img
def setup_header(ts_info): start_today = datetime.combine(date.today(), time.min) end_today = datetime.combine(date.today(), time.max) times = iter_date_range(start_today, end_today, ts_info['interval'] * 60) times = [x.strftime("%H:%M:%S") for x in times] return times
def __init__(self, ts_path=None, version=None, interval=None, start=None, end=None, start_hour=None, end_hour=None, ignored_timestamps=[]): """Class to got back and forth on a TimeStream Use This class when you need to traverse the timestream both forwards and backwards in time. Args: version(int): Passed directly to TimeStream interval(int): Interval between time stamps start(datetime): Start of time stream end(datetime): End of time stream start_hour(datetime): Starting hour within every day of time stream end_hour(datetime): Ending hour within every day of time stream ignored_timestamps(list): List of ignore time stamps.0 Attributes: _timestamps(list): List of strings that index all existing image files in this timestream _offset(int): Current offset within _timestamps. """ super(TimeStreamTraverser, self).__init__(version=version) self.load(ts_path) self._offset = 0 self._timestamps = [] # FIXME: Following is practically equal to # TimeStream.iter_by_timepoints. if not start or start < self.start_datetime: start = self.start_datetime if not end or end > self.end_datetime: end = self.end_datetime if not interval: interval = self.interval # fix hour range if given if start_hour is not None: start = dt.datetime.combine(start.date(), start_hour) if end_hour is not None: end = dt.datetime.combine(end.date(), end_hour) # iterate thru times for time in iter_date_range(start, end, interval): # skip images in ignored_timestamps if ts_format_date(time) in ignored_timestamps: continue # apply hour range if given if start_hour is not None: hrstart = dt.datetime.combine(time.date(), start_hour) if time < hrstart: continue if end_hour is not None: hrend = dt.datetime.combine(time.date(), end_hour) if time > hrend: continue # If path exists add index to _timestamps relpath = _ts_date_to_path(self.name, self.extension, time, 0) img_path = path.join(self.path, relpath) if path.exists(img_path): self._timestamps.append(time)