def events_summary():
has_clip = request.args.get("has_clip", type=int)
has_snapshot = request.args.get("has_snapshot", type=int)
clauses = []
if not has_clip is None:
clauses.append((Event.has_clip == has_clip))
if not has_snapshot is None:
clauses.append((Event.has_snapshot == has_snapshot))
if len(clauses) == 0:
clauses.append((1 == 1))
groups = (Event.select(
Event.camera,
Event.label,
fn.strftime("%Y-%m-%d",
fn.datetime(Event.start_time, "unixepoch",
"localtime")).alias("day"),
Event.zones,
fn.COUNT(Event.id).alias("count"),
).where(reduce(operator.and_, clauses)).group_by(
Event.camera,
Event.label,
fn.strftime("%Y-%m-%d",
fn.datetime(Event.start_time, "unixepoch", "localtime")),
Event.zones,
))
return jsonify([e for e in groups.dicts()])
def events():
limit = request.args.get("limit", 100)
camera = request.args.get("camera")
label = request.args.get("label")
zone = request.args.get("zone")
after = request.args.get("after", type=float)
before = request.args.get("before", type=float)
has_clip = request.args.get("has_clip", type=int)
has_snapshot = request.args.get("has_snapshot", type=int)
include_thumbnails = request.args.get("include_thumbnails", default=1, type=int)
clauses = []
excluded_fields = []
if camera:
clauses.append((Event.camera == camera))
if label:
clauses.append((Event.label == label))
if zone:
clauses.append((Event.zones.cast("text") % f'*"{zone}"*'))
if after:
clauses.append((Event.start_time >= after))
if before:
clauses.append((Event.start_time <= before))
if not has_clip is None:
clauses.append((Event.has_clip == has_clip))
if not has_snapshot is None:
clauses.append((Event.has_snapshot == has_snapshot))
if not include_thumbnails:
excluded_fields.append(Event.thumbnail)
if len(clauses) == 0:
clauses.append((True))
events = (
Event.select()
.where(reduce(operator.and_, clauses))
.order_by(Event.start_time.desc())
.limit(limit)
)
return jsonify([model_to_dict(e, exclude=excluded_fields) for e in events])
def events():
limit = request.args.get('limit', 100)
camera = request.args.get('camera')
label = request.args.get('label')
zone = request.args.get('zone')
after = request.args.get('after', type=float)
before = request.args.get('before', type=float)
has_clip = request.args.get('has_clip', type=int)
has_snapshot = request.args.get('has_snapshot', type=int)
include_thumbnails = request.args.get('include_thumbnails',
default=1,
type=int)
clauses = []
excluded_fields = []
if camera:
clauses.append((Event.camera == camera))
if label:
clauses.append((Event.label == label))
if zone:
clauses.append((Event.zones.cast('text') % f"*\"{zone}\"*"))
if after:
clauses.append((Event.start_time >= after))
if before:
clauses.append((Event.start_time <= before))
if not has_clip is None:
clauses.append((Event.has_clip == has_clip))
if not has_snapshot is None:
clauses.append((Event.has_snapshot == has_snapshot))
if not include_thumbnails:
excluded_fields.append(Event.thumbnail)
if len(clauses) == 0:
clauses.append((1 == 1))
events = (Event.select().where(reduce(operator.and_, clauses)).order_by(
Event.start_time.desc()).limit(limit))
return jsonify([model_to_dict(e, exclude=excluded_fields) for e in events])
def move_files(self):
cache_files = sorted([
d for d in os.listdir(CACHE_DIR)
if os.path.isfile(os.path.join(CACHE_DIR, d))
and d.endswith(".mp4") and not d.startswith("clip_")
])
files_in_use = []
for process in psutil.process_iter():
try:
if process.name() != "ffmpeg":
continue
flist = process.open_files()
if flist:
for nt in flist:
if nt.path.startswith(CACHE_DIR):
files_in_use.append(nt.path.split("/")[-1])
except:
continue
# group recordings by camera
grouped_recordings = defaultdict(list)
for f in cache_files:
# Skip files currently in use
if f in files_in_use:
continue
cache_path = os.path.join(CACHE_DIR, f)
basename = os.path.splitext(f)[0]
camera, date = basename.rsplit("-", maxsplit=1)
start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S")
grouped_recordings[camera].append({
"cache_path": cache_path,
"start_time": start_time,
})
# delete all cached files past the most recent 5
keep_count = 5
for camera in grouped_recordings.keys():
if len(grouped_recordings[camera]) > keep_count:
to_remove = grouped_recordings[camera][:-keep_count]
for f in to_remove:
Path(f["cache_path"]).unlink(missing_ok=True)
self.end_time_cache.pop(f["cache_path"], None)
grouped_recordings[camera] = grouped_recordings[camera][
-keep_count:]
for camera, recordings in grouped_recordings.items():
# clear out all the recording info for old frames
while (len(self.recordings_info[camera]) > 0
and self.recordings_info[camera][0][0] <
recordings[0]["start_time"].timestamp()):
self.recordings_info[camera].pop(0)
# get all events with the end time after the start of the oldest cache file
# or with end_time None
events: Event = (Event.select().where(
Event.camera == camera,
(Event.end_time == None)
| (Event.end_time >= recordings[0]["start_time"].timestamp()),
Event.has_clip,
).order_by(Event.start_time))
for r in recordings:
cache_path = r["cache_path"]
start_time = r["start_time"]
# Just delete files if recordings are turned off
if (not camera in self.config.cameras
or not self.config.cameras[camera].record.enabled):
Path(cache_path).unlink(missing_ok=True)
self.end_time_cache.pop(cache_path, None)
continue
if cache_path in self.end_time_cache:
end_time, duration = self.end_time_cache[cache_path]
else:
ffprobe_cmd = [
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
f"{cache_path}",
]
p = sp.run(ffprobe_cmd, capture_output=True)
if p.returncode == 0:
duration = float(p.stdout.decode().strip())
end_time = start_time + datetime.timedelta(
seconds=duration)
self.end_time_cache[cache_path] = (end_time, duration)
else:
logger.warning(
f"Discarding a corrupt recording segment: {f}")
Path(cache_path).unlink(missing_ok=True)
continue
# if cached file's start_time is earlier than the retain days for the camera
if start_time <= (
(datetime.datetime.now() - datetime.timedelta(
days=self.config.cameras[camera].record.retain.days))):
# if the cached segment overlaps with the events:
overlaps = False
for event in events:
# if the event starts in the future, stop checking events
# and remove this segment
if event.start_time > end_time.timestamp():
overlaps = False
Path(cache_path).unlink(missing_ok=True)
self.end_time_cache.pop(cache_path, None)
break
# if the event is in progress or ends after the recording starts, keep it
# and stop looking at events
if (event.end_time is None
or event.end_time >= start_time.timestamp()):
overlaps = True
break
if overlaps:
record_mode = self.config.cameras[
camera].record.events.retain.mode
# move from cache to recordings immediately
self.store_segment(
camera,
start_time,
end_time,
duration,
cache_path,
record_mode,
)
# else retain days includes this segment
else:
record_mode = self.config.cameras[
camera].record.retain.mode
self.store_segment(camera, start_time, end_time, duration,
cache_path, record_mode)
def expire_recordings(self):
logger.debug("Start expire recordings (new).")
logger.debug("Start deleted cameras.")
# Handle deleted cameras
expire_days = self.config.record.retain.days
expire_before = (datetime.datetime.now() -
datetime.timedelta(days=expire_days)).timestamp()
no_camera_recordings: Recordings = Recordings.select().where(
Recordings.camera.not_in(list(self.config.cameras.keys())),
Recordings.end_time < expire_before,
)
deleted_recordings = set()
for recording in no_camera_recordings:
Path(recording.path).unlink(missing_ok=True)
deleted_recordings.add(recording.id)
logger.debug(f"Expiring {len(deleted_recordings)} recordings")
Recordings.delete().where(
Recordings.id << deleted_recordings).execute()
logger.debug("End deleted cameras.")
logger.debug("Start all cameras.")
for camera, config in self.config.cameras.items():
logger.debug(f"Start camera: {camera}.")
# When deleting recordings without events, we have to keep at LEAST the configured max clip duration
min_end = (datetime.datetime.now() - datetime.timedelta(
seconds=config.record.events.max_seconds)).timestamp()
expire_days = config.record.retain.days
expire_before = (datetime.datetime.now() -
datetime.timedelta(days=expire_days)).timestamp()
expire_date = min(min_end, expire_before)
# Get recordings to check for expiration
recordings: Recordings = (Recordings.select().where(
Recordings.camera == camera,
Recordings.end_time < expire_date,
).order_by(Recordings.start_time))
# Get all the events to check against
events: Event = (
Event.select().where(
Event.camera == camera,
# need to ensure segments for all events starting
# before the expire date are included
Event.start_time < expire_date,
Event.has_clip,
).order_by(Event.start_time).objects())
# loop over recordings and see if they overlap with any non-expired events
# TODO: expire segments based on segment stats according to config
event_start = 0
deleted_recordings = set()
for recording in recordings.objects().iterator():
keep = False
# Now look for a reason to keep this recording segment
for idx in range(event_start, len(events)):
event = events[idx]
# if the event starts in the future, stop checking events
# and let this recording segment expire
if event.start_time > recording.end_time:
keep = False
break
# if the event is in progress or ends after the recording starts, keep it
# and stop looking at events
if event.end_time is None or event.end_time >= recording.start_time:
keep = True
break
# if the event ends before this recording segment starts, skip
# this event and check the next event for an overlap.
# since the events and recordings are sorted, we can skip events
# that end before the previous recording segment started on future segments
if event.end_time < recording.start_time:
event_start = idx
# Delete recordings outside of the retention window or based on the retention mode
if (not keep or
(config.record.events.retain.mode == RetainModeEnum.motion
and recording.motion == 0)
or (config.record.events.retain.mode
== RetainModeEnum.active_objects
and recording.objects == 0)):
Path(recording.path).unlink(missing_ok=True)
deleted_recordings.add(recording.id)
logger.debug(f"Expiring {len(deleted_recordings)} recordings")
Recordings.delete().where(
Recordings.id << deleted_recordings).execute()
logger.debug(f"End camera: {camera}.")
logger.debug("End all cameras.")
logger.debug("End expire recordings (new).")
def expire(self, media_type):
## Expire events from unlisted cameras based on the global config
if media_type == 'clips':
retain_config = self.config.clips.retain
file_extension = "mp4"
update_params = {"has_clip": False}
else:
retain_config = self.config.snapshots.retain
file_extension = "jpg"
update_params = {"has_snapshot": False}
distinct_labels = (Event.select(Event.label).where(
Event.camera.not_in(self.camera_keys)).distinct())
# loop over object types in db
for l in distinct_labels:
# get expiration time for this label
expire_days = retain_config.objects.get(l.label,
retain_config.default)
expire_after = (datetime.datetime.now() -
datetime.timedelta(days=expire_days)).timestamp()
# grab all events after specific time
expired_events = Event.select().where(
Event.camera.not_in(self.camera_keys),
Event.start_time < expire_after,
Event.label == l.label,
)
# delete the media from disk
for event in expired_events:
media_name = f"{event.camera}-{event.id}"
media_path = Path(
f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}")
media_path.unlink(missing_ok=True)
# update the clips attribute for the db entry
update_query = Event.update(update_params).where(
Event.camera.not_in(self.camera_keys),
Event.start_time < expire_after,
Event.label == l.label,
)
update_query.execute()
## Expire events from cameras based on the camera config
for name, camera in self.config.cameras.items():
if media_type == 'clips':
retain_config = camera.clips.retain
else:
retain_config = camera.snapshots.retain
# get distinct objects in database for this camera
distinct_labels = (Event.select(
Event.label).where(Event.camera == name).distinct())
# loop over object types in db
for l in distinct_labels:
# get expiration time for this label
expire_days = retain_config.objects.get(
l.label, retain_config.default)
expire_after = (
datetime.datetime.now() -
datetime.timedelta(days=expire_days)).timestamp()
# grab all events after specific time
expired_events = Event.select().where(
Event.camera == name,
Event.start_time < expire_after,
Event.label == l.label,
)
# delete the grabbed clips from disk
for event in expired_events:
media_name = f"{event.camera}-{event.id}"
media_path = Path(
f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}"
)
media_path.unlink(missing_ok=True)
# update the clips attribute for the db entry
update_query = Event.update(update_params).where(
Event.camera == name,
Event.start_time < expire_after,
Event.label == l.label,
)
update_query.execute()
