Ejemplo n.º 1
0
def main(opts):
    setup_module_logging(logging.ERROR)
    ts_name = opts["-i"]
    out_fh = open(opts['-o'], 'w')
    ts_info = ts_get_manifest(ts_name)
    times = setup_header(ts_info)
    out_csv = csv.writer(out_fh)
    header = ["Date", ]
    header.extend(times)
    out_csv.writerow(header)
    ts = TimeStream()
    ts.load(ts_name)
    res_dict = {}
    print("Collecting image data for {}...".format(ts_name))
    count = 0
    for img in ts.iter_by_timepoints():
        if count % 5 == 0:
            print("Processed {} images!".format(count), end='\r')
            sys.stdout.flush()
        count += 1
        img_dt = img.datetime
        try:
            res_dict[img_dt.date()][img_dt.time().isoformat()] = 1
        except KeyError:
            res_dict[img_dt.date()] = {img_dt.time().isoformat(): 1}
    print("Processed {} images!".format(count))
    print("Done collecting image sums, now making the table")
    for this_date, times in sorted(res_dict.items()):
        row = []
        row.append(this_date.isoformat())
        start_today = datetime.combine(this_date.today(), time.min)
        end_today = datetime.combine(this_date.today(), time.max)
        all_times = iter_date_range(start_today, end_today,
                                    ts_info['interval'] * 60)
        for timepoint in all_times:
            try:
                row.append(times[timepoint.time().isoformat()])
            except KeyError:
                row.append("0")
        out_csv.writerow(row)
    print("All done!")
Ejemplo n.º 2
0
    def __init__(self, ts_path=None, version=None, interval=None,
                 start=None, end=None, start_hour=None, end_hour=None,
                 ignore_ts=[], existing_ts=[], err_on_access=False):
        """Class to got back and forth on a TimeStream

        Use This class when you need to traverse the timestream both forwards
        and backwards in time.

        Args:
          version(int): Passed directly to TimeStream
          interval(int): Interval between time stamps
          start(datetime): Start of time stream
          end(datetime): End of time stream
          start_hour(datetime): Starting hour within every day of time stream
          end_hour(datetime): Ending hour within every day of time stream
          ignore_ts(list): Timestamps to be ignored.
          existing_ts(list): existing timestamps that should not be recalculated.
          err_on_access: If false we make all error checks silently in __init__.
                         If true we raise errors when accessing imgs.

        Attributes:
          _timestamps(list): List of strings that index all existing image files
            in this timestream
          _offset(int): Current offset within _timestamps.

        """
        super(TimeStreamTraverser, self).__init__(version=version)
        self.load(ts_path)

        self._err_on_access = err_on_access
        self._ignore_ts = ignore_ts
        self._existing_ts = existing_ts

        self._offset = 0
        self._timestamps = []
        # FIXME: Following is practically equal to
        # TimeStream.iter_by_timepoints.
        if not start or start < self.start_datetime:
            start = self.start_datetime
        if not end or end > self.end_datetime:
            end = self.end_datetime
        if not interval:
            interval = self.interval

        # fix hour range if given
        if start_hour is not None:
            start = dt.datetime.combine(start.date(), start_hour)
        if end_hour is not None:
            end = dt.datetime.combine(end.date(), end_hour)

        # iterate threw times
        for time in iter_date_range(start, end, interval):
            # apply hour range if given
            if start_hour is not None:
                hrstart = dt.datetime.combine(time.date(), start_hour)
                if time < hrstart:
                    continue
            if end_hour is not None:
                hrend = dt.datetime.combine(time.date(), end_hour)
                if time > hrend:
                    continue

            # skip images in ignore_ts
            if ts_format_date(time) in self._ignore_ts:
                if not self._err_on_access:
                    continue

            # skip images in existing_ts
            if ts_format_date(time) in self._existing_ts:
                if not self._err_on_access:
                    continue

            # Do not add if path dosn't exist
            relpath = _ts_date_to_path(self.name, self.extension, time, 0)
            img_path = path.join(self.path, relpath)
            if not path.exists(img_path):
                if not self._err_on_access:
                    continue

            self._timestamps.append(time)
Ejemplo n.º 3
0
    def iter_by_timepoints(self, remove_gaps=True, start=None, end=None,
                           interval=None, start_hour=None, end_hour=None,
                           ignored_timestamps=[]):
        """
        Iterate over a TimeStream in chronological order, yielding a
        TimeStreamImage instance for each timepoint. If ``remove_gaps`` is
        False, yield None for missing images.
        """
        if not start or start < self.start_datetime:
            start = self.start_datetime
        if not end or end > self.end_datetime:
            end = self.end_datetime
        if not interval:
            interval = self.interval

        # fix hour range if given
        if start_hour is not None:
            start = dt.datetime.combine(start.date(), start_hour)
        if end_hour is not None:
            end = dt.datetime.combine(end.date(), end_hour)

        # iterate thru times
        for time in iter_date_range(start, end, interval):
            # skip images in ignored_timestamps
            if ts_format_date(time) in ignored_timestamps:
                LOG.info("Skip processing data at {}".format(time))
                continue

            # apply hour range if given
            if start_hour is not None:
                hrstart = dt.datetime.combine(time.date(), start_hour)
                if time < hrstart:
                    continue
            if end_hour is not None:
                hrend = dt.datetime.combine(time.date(), end_hour)
                if time > hrend:
                    continue

            # Format the path below the ts root
            relpath = _ts_date_to_path(self.name, self.extension, time, 0)
            # Join to make "absolute" path, i.e. path including ts_path
            img_path = path.join(self.path, relpath)
            # not-so-silently fail if we can't find the image
            if path.exists(img_path):
                LOG.debug("Image at {} in {} is {}.".format(time, self.path,
                                                            img_path))
            else:
                LOG.debug("Expected image {} at {} did not exist.".format(
                    img_path, time, self.path))
                img_path = None
            if remove_gaps and img_path is None:
                continue
            elif img_path is None:
                img = TimeStreamImage(dt=time)
                img.pixels = np.array([])
                yield img
            else:
                img = self.load_pickled_image(time)
                if img is None:
                    img = TimeStreamImage(dt=time)
                img.parent_timestream = self
                img.path = img_path

                try:
                    img_date = ts_format_date(img.datetime)
                    img.data = self.image_data[img_date]
                except KeyError:
                    img.data = {}
                yield img
Ejemplo n.º 4
0
def setup_header(ts_info):
    start_today = datetime.combine(date.today(), time.min)
    end_today = datetime.combine(date.today(), time.max)
    times = iter_date_range(start_today, end_today, ts_info['interval'] * 60)
    times = [x.strftime("%H:%M:%S") for x in times]
    return times
Ejemplo n.º 5
0
    def __init__(self, ts_path=None, version=None, interval=None,
                 start=None, end=None, start_hour=None, end_hour=None,
                 ignored_timestamps=[]):
        """Class to got back and forth on a TimeStream

        Use This class when you need to traverse the timestream both forwards
        and backwards in time.

        Args:
          version(int): Passed directly to TimeStream
          interval(int): Interval between time stamps
          start(datetime): Start of time stream
          end(datetime): End of time stream
          start_hour(datetime): Starting hour within every day of time stream
          end_hour(datetime): Ending hour within every day of time stream
          ignored_timestamps(list): List of ignore time stamps.0

        Attributes:
          _timestamps(list): List of strings that index all existing image files
            in this timestream
          _offset(int): Current offset within _timestamps.

        """
        super(TimeStreamTraverser, self).__init__(version=version)
        self.load(ts_path)

        self._offset = 0
        self._timestamps = []
        # FIXME: Following is practically equal to
        # TimeStream.iter_by_timepoints.
        if not start or start < self.start_datetime:
            start = self.start_datetime
        if not end or end > self.end_datetime:
            end = self.end_datetime
        if not interval:
            interval = self.interval

        # fix hour range if given
        if start_hour is not None:
            start = dt.datetime.combine(start.date(), start_hour)
        if end_hour is not None:
            end = dt.datetime.combine(end.date(), end_hour)

        # iterate thru times
        for time in iter_date_range(start, end, interval):
            # skip images in ignored_timestamps
            if ts_format_date(time) in ignored_timestamps:
                continue

            # apply hour range if given
            if start_hour is not None:
                hrstart = dt.datetime.combine(time.date(), start_hour)
                if time < hrstart:
                    continue
            if end_hour is not None:
                hrend = dt.datetime.combine(time.date(), end_hour)
                if time > hrend:
                    continue

            # If path exists add index to _timestamps
            relpath = _ts_date_to_path(self.name, self.extension, time, 0)
            img_path = path.join(self.path, relpath)
            if path.exists(img_path):
                self._timestamps.append(time)