def events_summary(): has_clip = request.args.get("has_clip", type=int) has_snapshot = request.args.get("has_snapshot", type=int) clauses = [] if not has_clip is None: clauses.append((Event.has_clip == has_clip)) if not has_snapshot is None: clauses.append((Event.has_snapshot == has_snapshot)) if len(clauses) == 0: clauses.append((1 == 1)) groups = (Event.select( Event.camera, Event.label, fn.strftime("%Y-%m-%d", fn.datetime(Event.start_time, "unixepoch", "localtime")).alias("day"), Event.zones, fn.COUNT(Event.id).alias("count"), ).where(reduce(operator.and_, clauses)).group_by( Event.camera, Event.label, fn.strftime("%Y-%m-%d", fn.datetime(Event.start_time, "unixepoch", "localtime")), Event.zones, )) return jsonify([e for e in groups.dicts()])
def events(): limit = request.args.get("limit", 100) camera = request.args.get("camera") label = request.args.get("label") zone = request.args.get("zone") after = request.args.get("after", type=float) before = request.args.get("before", type=float) has_clip = request.args.get("has_clip", type=int) has_snapshot = request.args.get("has_snapshot", type=int) include_thumbnails = request.args.get("include_thumbnails", default=1, type=int) clauses = [] excluded_fields = [] if camera: clauses.append((Event.camera == camera)) if label: clauses.append((Event.label == label)) if zone: clauses.append((Event.zones.cast("text") % f'*"{zone}"*')) if after: clauses.append((Event.start_time >= after)) if before: clauses.append((Event.start_time <= before)) if not has_clip is None: clauses.append((Event.has_clip == has_clip)) if not has_snapshot is None: clauses.append((Event.has_snapshot == has_snapshot)) if not include_thumbnails: excluded_fields.append(Event.thumbnail) if len(clauses) == 0: clauses.append((True)) events = ( Event.select() .where(reduce(operator.and_, clauses)) .order_by(Event.start_time.desc()) .limit(limit) ) return jsonify([model_to_dict(e, exclude=excluded_fields) for e in events])
def events(): limit = request.args.get('limit', 100) camera = request.args.get('camera') label = request.args.get('label') zone = request.args.get('zone') after = request.args.get('after', type=float) before = request.args.get('before', type=float) has_clip = request.args.get('has_clip', type=int) has_snapshot = request.args.get('has_snapshot', type=int) include_thumbnails = request.args.get('include_thumbnails', default=1, type=int) clauses = [] excluded_fields = [] if camera: clauses.append((Event.camera == camera)) if label: clauses.append((Event.label == label)) if zone: clauses.append((Event.zones.cast('text') % f"*\"{zone}\"*")) if after: clauses.append((Event.start_time >= after)) if before: clauses.append((Event.start_time <= before)) if not has_clip is None: clauses.append((Event.has_clip == has_clip)) if not has_snapshot is None: clauses.append((Event.has_snapshot == has_snapshot)) if not include_thumbnails: excluded_fields.append(Event.thumbnail) if len(clauses) == 0: clauses.append((1 == 1)) events = (Event.select().where(reduce(operator.and_, clauses)).order_by( Event.start_time.desc()).limit(limit)) return jsonify([model_to_dict(e, exclude=excluded_fields) for e in events])
def move_files(self): cache_files = sorted([ d for d in os.listdir(CACHE_DIR) if os.path.isfile(os.path.join(CACHE_DIR, d)) and d.endswith(".mp4") and not d.startswith("clip_") ]) files_in_use = [] for process in psutil.process_iter(): try: if process.name() != "ffmpeg": continue flist = process.open_files() if flist: for nt in flist: if nt.path.startswith(CACHE_DIR): files_in_use.append(nt.path.split("/")[-1]) except: continue # group recordings by camera grouped_recordings = defaultdict(list) for f in cache_files: # Skip files currently in use if f in files_in_use: continue cache_path = os.path.join(CACHE_DIR, f) basename = os.path.splitext(f)[0] camera, date = basename.rsplit("-", maxsplit=1) start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S") grouped_recordings[camera].append({ "cache_path": cache_path, "start_time": start_time, }) # delete all cached files past the most recent 5 keep_count = 5 for camera in grouped_recordings.keys(): if len(grouped_recordings[camera]) > keep_count: to_remove = grouped_recordings[camera][:-keep_count] for f in to_remove: Path(f["cache_path"]).unlink(missing_ok=True) self.end_time_cache.pop(f["cache_path"], None) grouped_recordings[camera] = grouped_recordings[camera][ -keep_count:] for camera, recordings in grouped_recordings.items(): # clear out all the recording info for old frames while (len(self.recordings_info[camera]) > 0 and self.recordings_info[camera][0][0] < recordings[0]["start_time"].timestamp()): self.recordings_info[camera].pop(0) # get all events with the end time after the start of the oldest cache file # or with end_time None events: Event = (Event.select().where( Event.camera == camera, (Event.end_time == None) | (Event.end_time >= recordings[0]["start_time"].timestamp()), Event.has_clip, ).order_by(Event.start_time)) for r in recordings: cache_path = r["cache_path"] start_time = r["start_time"] # Just delete files if recordings are turned off if (not camera in self.config.cameras or not self.config.cameras[camera].record.enabled): Path(cache_path).unlink(missing_ok=True) self.end_time_cache.pop(cache_path, None) continue if cache_path in self.end_time_cache: end_time, duration = self.end_time_cache[cache_path] else: ffprobe_cmd = [ "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", f"{cache_path}", ] p = sp.run(ffprobe_cmd, capture_output=True) if p.returncode == 0: duration = float(p.stdout.decode().strip()) end_time = start_time + datetime.timedelta( seconds=duration) self.end_time_cache[cache_path] = (end_time, duration) else: logger.warning( f"Discarding a corrupt recording segment: {f}") Path(cache_path).unlink(missing_ok=True) continue # if cached file's start_time is earlier than the retain days for the camera if start_time <= ( (datetime.datetime.now() - datetime.timedelta( days=self.config.cameras[camera].record.retain.days))): # if the cached segment overlaps with the events: overlaps = False for event in events: # if the event starts in the future, stop checking events # and remove this segment if event.start_time > end_time.timestamp(): overlaps = False Path(cache_path).unlink(missing_ok=True) self.end_time_cache.pop(cache_path, None) break # if the event is in progress or ends after the recording starts, keep it # and stop looking at events if (event.end_time is None or event.end_time >= start_time.timestamp()): overlaps = True break if overlaps: record_mode = self.config.cameras[ camera].record.events.retain.mode # move from cache to recordings immediately self.store_segment( camera, start_time, end_time, duration, cache_path, record_mode, ) # else retain days includes this segment else: record_mode = self.config.cameras[ camera].record.retain.mode self.store_segment(camera, start_time, end_time, duration, cache_path, record_mode)
def expire_recordings(self): logger.debug("Start expire recordings (new).") logger.debug("Start deleted cameras.") # Handle deleted cameras expire_days = self.config.record.retain.days expire_before = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() no_camera_recordings: Recordings = Recordings.select().where( Recordings.camera.not_in(list(self.config.cameras.keys())), Recordings.end_time < expire_before, ) deleted_recordings = set() for recording in no_camera_recordings: Path(recording.path).unlink(missing_ok=True) deleted_recordings.add(recording.id) logger.debug(f"Expiring {len(deleted_recordings)} recordings") Recordings.delete().where( Recordings.id << deleted_recordings).execute() logger.debug("End deleted cameras.") logger.debug("Start all cameras.") for camera, config in self.config.cameras.items(): logger.debug(f"Start camera: {camera}.") # When deleting recordings without events, we have to keep at LEAST the configured max clip duration min_end = (datetime.datetime.now() - datetime.timedelta( seconds=config.record.events.max_seconds)).timestamp() expire_days = config.record.retain.days expire_before = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() expire_date = min(min_end, expire_before) # Get recordings to check for expiration recordings: Recordings = (Recordings.select().where( Recordings.camera == camera, Recordings.end_time < expire_date, ).order_by(Recordings.start_time)) # Get all the events to check against events: Event = ( Event.select().where( Event.camera == camera, # need to ensure segments for all events starting # before the expire date are included Event.start_time < expire_date, Event.has_clip, ).order_by(Event.start_time).objects()) # loop over recordings and see if they overlap with any non-expired events # TODO: expire segments based on segment stats according to config event_start = 0 deleted_recordings = set() for recording in recordings.objects().iterator(): keep = False # Now look for a reason to keep this recording segment for idx in range(event_start, len(events)): event = events[idx] # if the event starts in the future, stop checking events # and let this recording segment expire if event.start_time > recording.end_time: keep = False break # if the event is in progress or ends after the recording starts, keep it # and stop looking at events if event.end_time is None or event.end_time >= recording.start_time: keep = True break # if the event ends before this recording segment starts, skip # this event and check the next event for an overlap. # since the events and recordings are sorted, we can skip events # that end before the previous recording segment started on future segments if event.end_time < recording.start_time: event_start = idx # Delete recordings outside of the retention window or based on the retention mode if (not keep or (config.record.events.retain.mode == RetainModeEnum.motion and recording.motion == 0) or (config.record.events.retain.mode == RetainModeEnum.active_objects and recording.objects == 0)): Path(recording.path).unlink(missing_ok=True) deleted_recordings.add(recording.id) logger.debug(f"Expiring {len(deleted_recordings)} recordings") Recordings.delete().where( Recordings.id << deleted_recordings).execute() logger.debug(f"End camera: {camera}.") logger.debug("End all cameras.") logger.debug("End expire recordings (new).")
def expire(self, media_type): ## Expire events from unlisted cameras based on the global config if media_type == 'clips': retain_config = self.config.clips.retain file_extension = "mp4" update_params = {"has_clip": False} else: retain_config = self.config.snapshots.retain file_extension = "jpg" update_params = {"has_snapshot": False} distinct_labels = (Event.select(Event.label).where( Event.camera.not_in(self.camera_keys)).distinct()) # loop over object types in db for l in distinct_labels: # get expiration time for this label expire_days = retain_config.objects.get(l.label, retain_config.default) expire_after = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() # grab all events after specific time expired_events = Event.select().where( Event.camera.not_in(self.camera_keys), Event.start_time < expire_after, Event.label == l.label, ) # delete the media from disk for event in expired_events: media_name = f"{event.camera}-{event.id}" media_path = Path( f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") media_path.unlink(missing_ok=True) # update the clips attribute for the db entry update_query = Event.update(update_params).where( Event.camera.not_in(self.camera_keys), Event.start_time < expire_after, Event.label == l.label, ) update_query.execute() ## Expire events from cameras based on the camera config for name, camera in self.config.cameras.items(): if media_type == 'clips': retain_config = camera.clips.retain else: retain_config = camera.snapshots.retain # get distinct objects in database for this camera distinct_labels = (Event.select( Event.label).where(Event.camera == name).distinct()) # loop over object types in db for l in distinct_labels: # get expiration time for this label expire_days = retain_config.objects.get( l.label, retain_config.default) expire_after = ( datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() # grab all events after specific time expired_events = Event.select().where( Event.camera == name, Event.start_time < expire_after, Event.label == l.label, ) # delete the grabbed clips from disk for event in expired_events: media_name = f"{event.camera}-{event.id}" media_path = Path( f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}" ) media_path.unlink(missing_ok=True) # update the clips attribute for the db entry update_query = Event.update(update_params).where( Event.camera == name, Event.start_time < expire_after, Event.label == l.label, ) update_query.execute()