def end_predictions():
# Status level 4 on a video means that predictions have completed.
cursor.execute("""
UPDATE predict_progress
SET status=4
""")
con.commit()
def create_annotation_collection(model_name, user_id, video_id, concept_ids):
time_now = datetime.datetime.now().strftime(r"%y-%m-%d_%H:%M:%S")
collection_name = '_'.join([model_name, str(video_id), time_now])
description = f"By {model_name} on video {video_id} at {time_now}"
concept_names = pd_query(
"""
SELECT name
FROM concepts
WHERE id IN %s
""", (tuple(concept_ids),)
)['name'].tolist()
cursor.execute(
"""
INSERT INTO annotation_collection
(name, description, users, videos, concepts, tracking, conceptid)
VALUES (%s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(collection_name, description, [user_id], [video_id], concept_names,
False, concept_ids)
)
con.commit()
collection_id = int(cursor.fetchone()[0])
return collection_id
def upload_predict_progress(count, videoid, total_count, status):
'''
For updating the predict_progress psql database, which tracks prediction and
video generation status.
Arguments:
count - frame of video (or index of annotation) being processed
videoid - video being processed
total_count - total number of frames in the video (or number of predictions + annotations)
status - Indicates whether processing video or drawing annotation boxes
'''
print(
f'count: {count} total_count: {total_count} vid: {videoid} status: {status}'
)
if (count == 0):
cursor.execute(
'''
UPDATE predict_progress
SET framenum=%s, status=%s, totalframe=%s''', (
count,
status,
total_count,
))
con.commit()
return
if (total_count == count):
count = -1
cursor.execute(
'''
UPDATE predict_progress
SET framenum=%s''', (count, ))
con.commit()
def upload_annotation(frame, x1, x2, y1, y2, frame_num, conceptid, videoid,
videowidth, videoheight, userid, fps):
if userid is None:
raise ValueError("userid is None, can't upload annotations")
timeinvideo = frame_num / fps
no_box = str(videoid) + "_" + str(timeinvideo) + "_ai.png"
temp_file = str(uuid.uuid4()) + ".png"
cv2.imwrite(temp_file, frame)
s3.upload_file(temp_file,
config.S3_BUCKET,
config.S3_ANNOTATION_FOLDER + no_box,
ExtraArgs={'ContentType': 'image/png'})
os.system('rm ' + temp_file)
cursor.execute(
"""
INSERT INTO annotations (
videoid, userid, conceptid, timeinvideo, x1, y1, x2, y2,
videowidth, videoheight, dateannotated, image)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(int(videoid), int(userid), int(conceptid), timeinvideo, x1, y1, x2,
y2, videowidth, videoheight, datetime.datetime.now().date(), no_box))
annotation_id = cursor.fetchone()[0]
return annotation_id
def reset_model_params():
""" Reset the model_params table
"""
print("resetting model_params")
cursor.execute("""
Update model_params
SET epochs = 0, min_images=0, model='', annotation_collections=ARRAY[]:: integer[],
verified_only=null, include_tracking=null, version=0
WHERE option='train'
""")
con.commit()
def handle_annotation(prediction, frames, videoid, videoheight, videowidth,
userid, fps, collection_id):
frame = frames[int(prediction.frame_num)]
annotation_id = upload_annotation(
frame, *prediction.loc[['x1', 'x2', 'y1', 'y2', 'frame_num', 'label']],
videoid, videowidth, videoheight, userid, fps)
if collection_id is not None:
cursor.execute(
"""
INSERT INTO annotation_intermediate (id, annotationid)
VALUES (%s, %s)
""", (collection_id, annotation_id))
def reset_predict_params():
""" Reset the predict_params table
"""
print("resetting model_params")
cursor.execute(
"""
UPDATE predict_params
SET model='', userid=-1, concepts=ARRAY[]::integer[],
upload_annotations=false, videos=ARRAY[]::integer[],
version='0', create_collection=false
"""
)
con.commit()
def evaluate(video_id, model_username, concepts, upload_annotations=False,
userid=None, create_collection=False):
# file format: (video_id)_(model_name)-(version).mp4
if create_collection:
if not upload_annotations:
raise ValueError("cannot create new annotation collection if "
"annotations aren't uploaded")
if userid is None:
raise ValueError("userid is None, cannot create new collection")
collection_id = create_annotation_collection(model_username, userid, video_id, concepts)
else:
collection_id = None
filename = str(video_id) + "_" + model_username + ".mp4"
print("ai video filename: {0}".format(filename))
results, annotations = predict.predict_on_video(
video_id, config.WEIGHTS_PATH, concepts, filename, upload_annotations,
userid, collection_id)
if (results.empty):
return
username_split = model_username.split('-')
version = username_split[-1]
model_name = '-'.join(username_split[:-1])
# add the entry to ai_videos
cursor.execute('''
INSERT INTO ai_videos (name, videoid, version, model_name)
VALUES (%s, %s, %s, %s)''',
(filename, video_id, version, model_name)
)
con.commit()
print("done predicting")
metrics = score_predictions(
annotations, results, config.EVALUATION_IOU_THRESH, concepts
)
concept_counts = get_counts(results, annotations)
metrics = metrics.set_index("conceptid").join(concept_counts)
metrics.to_csv("metrics" + str(video_id) + ".csv")
# upload the data to s3 bucket
print("uploading to s3 folder")
s3.upload_file(
"metrics" + str(video_id) + ".csv",
config.S3_BUCKET,
config.S3_METRICS_FOLDER + filename.replace("mp4", "csv"),
ExtraArgs={"ContentType": "application/vnd.ms-excel"},
)
print(metrics)
con.commit()
def setup_predict_progress(verify_videos):
"""Reset the predict progress table for new predictions"""
# Just to be sure in case of web app not deleting the progress
# we clear the prediction progress table
cursor.execute("""DELETE FROM predict_progress""")
con.commit()
cursor.execute(
"""
INSERT INTO predict_progress (videoid, current_video, total_videos)
VALUES (%s, %s, %s)""",
(0, 0, len(verify_videos)),
)
con.commit()
def evaluate_videos(concepts,
verify_videos,
user_model,
upload_annotations=False,
userid=None,
create_collection=False):
""" Run evaluate on all the evaluation videos
"""
# We go one by one as multiprocessing ran into memory issues
for video_id in verify_videos:
cursor.execute(
f"""UPDATE predict_progress SET videoid = {video_id}, current_video = current_video + 1"""
)
con.commit()
evaluate(video_id, user_model, concepts, upload_annotations, userid,
create_collection)
end_predictions()
def create_model_user(new_version, model_params, user_model):
"""Insert a new user for this model version, then update the model_versions table
with the new model version
"""
print("creating new user, updating model_versions table")
cursor.execute(
"""
INSERT INTO users (username, password, admin)
VALUES (%s, 0, null)
RETURNING *""",
(user_model, ),
)
con.commit()
model_user_id = int(cursor.fetchone()[0])
# Update the model_versions table with the new user
cursor.execute(
""" INSERT INTO model_versions
(epochs,
min_images,
model,
annotation_collections,
verified_only,
include_tracking,
userid,
version,
timestamp)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) """,
(int(model_params["epochs"]), int(model_params["min_images"]),
model_params["model"], model_params["annotation_collections"],
bool(model_params["verified_only"]),
bool(model_params["include_tracking"]), model_user_id, new_version,
datetime.now()))
con.commit()
return model_user_id
RETURNING id
""",
(collection_name, description, [user_id], [video_id], concept_names,
False, concept_ids)
)
con.commit()
collection_id = int(cursor.fetchone()[0])
return collection_id
if __name__ == "__main__":
cursor.execute(
"""
SELECT * FROM models
LEFT JOIN users u ON u.id=userid
WHERE name=%s
""",
("testv3",),
)
model = cursor.fetchone()
video_id = 86
concepts = model[2]
userid = "270"
model_username = "******"
cursor.execute("""DELETE FROM predict_progress""")
con.commit()
cursor.execute(
"""
INSERT INTO predict_progress (videoid, current_video, total_videos)
def delete_model_user(model_user_id):
cursor.execute("""DELETE FROM model_versions
WHERE userid=%s""", (model_user_id, ))
cursor.execute("""DELETE FROM users
WHERE id=%s""", (model_user_id, ))
con.commit()
def _get_annotations(collection_ids, verified_only, include_tracking,
verify_videos, concepts, min_examples):
# Query that gets all annotations for given concepts
# making sure that any tracking annotations originated from good users
tracking_user = cursor.execute(
"""SELECT id FROM users WHERE username = '******'""")
tracking_uid = cursor.fetchone()[0]
annotations_query = r'''
WITH collection AS (SELECT
A.id,
image,
userid,
videoid,
videowidth,
videoheight,
conceptid,
x1, x2, y1, y2,
speed,
ROUND(fps * timeinvideo) as frame_num,
verifiedby
FROM
annotation_intermediate inter
LEFT JOIN
annotations a ON a.id=inter.annotationid
LEFT JOIN
videos ON videos.id=videoid
WHERE inter.id = ANY(%s) AND a.videoid <> ANY(%s)
'''
if verified_only:
annotations_query += """ AND a.verifiedby IS NOT NULL"""
# Filter collection so each concept has min_example annotations
annotations_query += r'''
),
filteredCollection AS (
SELECT
*
FROM (
SELECT
ROW_NUMBER() OVER (
PARTITION BY
conceptid
ORDER BY userid=32, verifiedby IS NULL) AS r,
c.*
FROM
collection c) t
WHERE
t.r <= (%s)
)
'''
# Add annotations that exist in the same frame
annotations_query += r'''
SELECT
A.id,
image,
userid,
videoid,
videowidth,
videoheight,
conceptid,
x1, x2, y1, y2,
speed,
priority,
ROUND(fps * timeinvideo) as frame_num,
verifiedby
FROM
annotations a
LEFT JOIN
videos ON videos.id=videoid
WHERE
ABS(x2-x1)>25 AND ABS(y2-y1)>25 AND
x1>=0 AND x1<videowidth AND
x2>0 AND x2<=videowidth AND
y1>=0 AND y1<videowidth AND
y2>0 AND y2<=videowidth AND
EXISTS (
SELECT
1
FROM
filteredCollection c
WHERE
c.videoid=a.videoid
AND c.frame_num=ROUND(fps * timeinvideo))
'''
if not include_tracking:
annotations_query += f''' AND a.userid <> {tracking_uid}'''
return pd_query(annotations_query,
(collection_ids, verify_videos, min_examples))
count,
status,
total_count,
))
con.commit()
return
if (total_count == count):
count = -1
cursor.execute(
'''
UPDATE predict_progress
SET framenum=%s''', (count, ))
con.commit()
if __name__ == '__main__':
model_name = 'testV2'
s3.download_file(config.S3_BUCKET,
config.S3_WEIGHTS_FOLDER + model_name + '.h5',
config.WEIGHTS_PATH)
cursor.execute("SELECT * FROM MODELS WHERE name='" + model_name + "'")
model = cursor.fetchone()
videoid = 86
concepts = model[2]
predict_on_video(videoid, config.WEIGHTS_PATH, concepts)