def coco_to_custom_vision(stream, project_id, trainer, data_dir):
stream = stream | mp.as_list
tags = stream | mp.select_field('class_id') | mp.dedup() | mp.as_list
cv_tags = {tag: trainer.create_tag(project_id, tag) for tag in tags}
stream = (
stream
| mp.apply(['width', 'height', 'ground_truth'], 'ground_truth',
lambda x: x[2]
| mp.where(lambda box: (box['width'] >= x[0] * 0.1) and
(box['height'] >= x[1] * 0.1))
| mp.as_list)
| mp.filter('ground_truth', lambda x: len(x) > 0)
| mp.apply(
['width', 'height', 'ground_truth'], 'regions', lambda x: x[2]
| mp.select(lambda box: Region(tag_id=cv_tags[box['tag']].id,
left=box['x1'] / x[0],
top=box['y1'] / x[1],
width=box['width'] / x[0],
height=box['height'] / x[1]))
| mp.as_list)
| mp.apply(['filename', 'regions'], 'tagged_img',
lambda x: ImageFileCreateEntry(
name=x[0],
contents=open(join(data_dir, x[0]), mode="rb").read(),
regions=x[1]))
| mp.as_list)
tagged_images_with_regions = stream | mp.select_field(
'tagged_img') | mp.as_list
for i in range(0, len(tagged_images_with_regions), 50):
trainer.create_images_from_files(
project_id, images=tagged_images_with_regions[i:i + 50])
def apply_quantized_model(stream, data_dir, model, dest_field):
return (stream
| mp.apply(
'filename', dest_field + '_raw',
lambda x: model.predict_image(Image.open(join(data_dir, x))))
| mp.apply([dest_field + '_raw', 'width', 'height'], dest_field,
lambda x: x[0]
| mp.select(lambda p: format_dict(p, x[1], x[2]))
| mp.as_list)
| mp.delfield([dest_field + '_raw']))
def videos_to_frames_pipe(data_dir,
ext='.avi',
target_ext='.allframes.npy',
classes=None,
min_size=128,
max_elements=13320):
"""
Creates and executes mPyPl pipe to load all videos from 'data_dir' (each subfolder is a separate class),
extracts all frames and saves them in numpy format
Parameters
----------
data_dir : str, required
The directory where all the vidoes are organised in subfolders (subfolder name=class name)
ext : str, optional
Extension of video files to search for, by default '.avi'
target_ext : str, optional
Target extension for video frames to be serialized to, by default '.allframes.npy'
classes : dict, optional
Dictionary with class names and numeral representations. Must match the folder names in 'data_dir'.
If set to 'None' it will automatically figure out classes based on folders structure in 'data_dir'.
Example {'Class1': 1, 'Class2': 2}
Defaults to 'None'
min_size : int, optional
Minimum size of frames based on the shorter edge, by default 128
max_elements : int, optional
Max elements for silly progress indicator, by default 13320
"""
(mp.get_datastream(data_dir, classes=classes, ext=ext)
| mp.apply('filename',
'clip',
lambda fn: VideoFileClip(fn),
eval_strategy=mp.EvalStrategies.Value)
| mp.apply('clip',
'clip',
lambda clip: clip.fx(vfx.resize, width=min_size)
if clip.w <= clip.h else clip.fx(vfx.resize, height=min_size),
eval_strategy=mp.EvalStrategies.Value)
| mp.apply('clip',
'allframes',
lambda c: np.asarray(list(c.iter_frames())),
eval_strategy=mp.EvalStrategies.Value)
| mp.iter('clip', close_clip)
| mp.delfield('clip')
| cachecomputex(
ext, target_ext, lambda x, nfn: np.save(nfn, x['allframes']),
lambda x, nfn: print("Skipping saving 'allframes' for {}".format(x[
'filename'])))
| mp.silly_progress(elements=max_elements)
| mp.execute)
def get_coco_stream(tags, ann_file, data_dir, threshold=0.1):
coco = COCO(ann_file)
catIds = coco.getCatIds(catNms=tags)
imgIds = sum([coco.getImgIds(catIds=catId) for catId in catIds], [])
stream = (coco.loadImgs(imgIds)
| mp.as_field('meta')
| mp.apply('meta', 'width', lambda x: x['width'])
| mp.apply('meta', 'height', lambda x: x['height'])
| mp.apply('meta', 'url', lambda x: x['coco_url'])
| mp.apply('meta', 'filename', lambda x: x['file_name'])
| mp.apply(
'meta', 'anns_ids', lambda x: coco.getAnnIds(
imgIds=x['id'], catIds=catIds, iscrowd=None))
| mp.apply('anns_ids', 'anns', lambda x: coco.loadAnns(x))
| mp.apply(
'anns', 'ground_truth', lambda x: x
| mp.select(lambda m: bbox_to_dict(
m['bbox'], coco.cats[m['category_id']]['name']))
| mp.as_list)
| mp.apply(
'ground_truth', 'class_id',
lambda x: most_common(x
| mp.select(lambda m: m['tag'])
| mp.as_list))
| mp.iter('meta', lambda x: coco.download(data_dir, [x['id']]))
| mp.delfield(['meta', 'anns_ids', 'anns']))
return stream
def precision_recall(data,
cls,
prob_threshold,
iou_treshold,
pred_field='predictions',
gt_field='ground_truth'):
TP = 0.0
TPFP = 0.0 # total positive results / pred
TPFN = 0.0 # total existing cases / rel
for obj in data:
ground_truth = (obj[gt_field]
| mp.where(lambda x: x['tag'] == cls)
| mp.as_list)
TPFN += len(ground_truth)
predictions = (
obj[pred_field]
|
mp.where(lambda x: x['tag'] == cls and x['prob'] > prob_threshold)
| mp.as_list)
for gt_box in ground_truth:
pred_boxes = (
predictions
| mp.apply(
['x1', 'y1', 'width', 'height'], 'iou',
lambda x: intersection_over_union(x, (gt_box['x1'], gt_box[
'y1'], gt_box['width'], gt_box['height'])))
| mp.filter('iou', lambda x: x < iou_treshold)
| mp.as_list)
if len(pred_boxes) > 0:
TP += 1
TPFP += len(pred_boxes)
return (float(TP > 0) if TPFP == 0 else TP / TPFP,
float(TP > 0) if TPFN == 0 else TP / TPFN)
def generate_img(data):
n = random.randint(3, 30)
x = (
data
| mp.pshuffle
| mp.take(n)
| mp.apply(
'filename', 'image',
lambda x: cv2.cvtColor(cv2.imread(os.path.splitext(x)[0] + '.jpg'),
cv2.COLOR_BGR2RGB))
| mp.apply(['image', 'descr'], 'face', transform)
#| mp.apply('face','facesmall',functools.partial(im_resize,size=(100,150)))
#| mp.select_field('facesmall')
| mp.select_field('face')
| mp.as_list)
return merge(x, np.random.random(len(x)))
def generate_img(data):
x = (data
| mp.pshuffle
| mp.take(args.mix)
| mp.apply(['image','landmarks'],'face',transform)
| mp.select_field('face')
| mp.as_list)
return merge(x,np.random.random(len(x)))
def load_moviepy_video(seq,
filename_field='filename',
video_field='video',
eval_strategy=mp.EvalStrategies.LazyMemoized):
return seq | mp.apply(filename_field,
video_field,
lambda x: VideoFileClip(x),
eval_strategy=eval_strategy)
def get_features_from_files(data_dir,
features_ext='.proc.c3d-avg.npy',
test_split=[],
classes=None,
max_elements=13320):
"""
Creates and executes mPyPl pipe to load feature vectors from serialized files and returns a preprocessed
data stream that can be further used with respect to train/test split and specific classes assigned to each element in the stream
Parameters
----------
data_dir : str, required
The directory where all the vidoes are organised in subfolders (subfolder name=class name)
features_ext : str, optional
Extension of serialized feature vectors, by default '.proc.c3d-avg.npy'
test_split : list, optional
List of filenames belonging to the test subset.
If empty then there will be no data in the test subset, by default []
classes : dict, optional
Dictionary with class names and numeral representations. Must match the folder names in 'data_dir'.
If set to 'None' it will automatically figure out classes based on folders structure in 'data_dir'.
Example: {'Class1': 1, 'Class2': 2}
Defaults to 'None'
max_elements : int, optional
Max elements for silly progress indicator, by default 13320
Returns
-------
list of mPyPl.mdict.mdict
List of dictionaries that can be used to access the data
"""
data = (mp.get_datastream(data_dir, classes=classes, ext=features_ext)
| mp.datasplit_by_pattern(test_pattern=test_split)
| mp.pshuffle
| mp.apply('filename', 'c3d_avg', lambda fn: np.load(fn))
| mp.silly_progress(elements=max_elements)
| mp.select_fields(['c3d_avg', 'class_id', 'split'])
| mp.as_list)
return data
from pipe import Pipe
from moviepy.editor import *
import numpy as np
import itertools
import cv2
import math
import matplotlib.pyplot as plt
import keras
from keras.models import Sequential
from keras.layers import *
from keras.regularizers import l2
test_names = (from_json(os.path.join(source_dir, 'matches.json'))
| mp.where(lambda x: 'Test' in x.keys() and int(x['Test']) > 0)
| mp.apply(['Id', 'Half'], 'pattern',
lambda x: "{}_{}_".format(x[0], x[1]))
| mp.select_field('pattern')
| mp.as_list)
no_frames = 126
data = (mp.get_datastream(data_dir, ext=".resized.mp4")
| mp.pshuffle
| datasplit_by_pattern(test_pattern=test_names)
| mp.apply('filename', 'vgg',
lambda x: np.load(x.replace('.resized.mp4', '.vgg.npy')))
| mp.apply('vgg', 'vggflat',
lambda x: np.reshape(x, (no_frames, -1, 1)))
| mp.take(500)
| mp.as_list)
max(0, y - y_expand):min(im_w, y + h + y_expand)]
def merge(images, wts=None):
res = np.zeros_like(images[0], dtype=np.float32)
if wts is None:
wts = np.ones(len(images))
wts /= np.sum(wts)
for n, i in enumerate(images):
res += wts[n] * i.astype(np.float32)
return res.astype(np.int32)
data = (mp.get_files(dir, ext='.json')
| mp.as_field('filename')
| mp.apply('filename', 'descr', loadjs)
| mp.unroll('descr')
| mp.filter(
'descr', lambda x: person_id == "" or
('candidates' in x and person_id in
[z['personId'] for z in x['candidates']]))
| mp.filter(
'descr', lambda x: abs(x['faceAttributes']['headPose']['yaw']) < 15
and abs(x['faceAttributes']['headPose']['pitch']) < 15)
| mp.filter(
'descr', lambda x: x['faceLandmarks']['pupilRight']['x'] - x[
'faceLandmarks']['pupilLeft']['x'] > 50)
| mp.as_list)
print("Found {} faces".format(len(data)))
def frames_to_features_pipe(data_dir,
mean_std,
model,
ext='.allframes.npy',
target_ext='.proc.c3d-avg.npy',
classes=None,
frames_per_clip=16,
frames_step=8,
batch_size=32,
max_elements=13320):
"""
Creates and executes mPyPl pipe to load all video frames, resize and crop them, preprocess,
run inferencing against Keras model and serialize the resulting feature vectors as npy format
Parameters
----------
data_dir : str, required
The directory where all the vidoes are organised in subfolders (subfolder name=class name)
mean_std : array, required
Array of per channel mean and std values used for preprocessing of frames.
Template: array[ [mean_R, mean_G, mean_B], [std_R, std_G, std_B] ]
Example: array[ [123, 112, 145], [60, 62, 64] ]
model : Keras model obj, required
Keras model object ready for running predictions
ext : str, optional
Extension of frames files to search for, by default '.allframes.npy'
target_ext : str, optional
Target extension for feature vectors to be serialized to, by default '.proc.c3d-avg.npy'
classes : dict, optional
Dictionary with class names and numeral representations. Must match the folder names in 'data_dir'.
If set to 'None' it will automatically figure out classes based on folders structure in 'data_dir'.
Example: {'Class1': 1, 'Class2': 2}
Defaults to 'None'
frames_per_clip : int, optional
When extracting smaller clips from longer video this defines the number of frames cut out from longer clip, by default 16
frames_step : int, optional
When extracting smaller clips from longer video this defines the step in number of frames, by default 8
batch_size : int, optional
Mini batch size used when pushing data to the model for scoring, by default 32
max_elements : int, optional
Max elements for silly progress indicator, by default 13320
"""
(mp.get_datastream(data_dir, classes=classes, ext=ext)
# load all frames for each video file
| mp.apply('filename',
'allframes',
lambda fn: np.load(fn),
eval_strategy=mp.EvalStrategies.OnDemand)
# cut each video into multiple shorter clips definded by frames_per_clip and frames_step parameters
| mp.apply('allframes',
'clips16-8',
lambda v: extract_clips(
v, frames_per_clip=frames_per_clip, step=frames_step),
eval_strategy=mp.EvalStrategies.OnDemand)
# center crop frames into 112x112
| mp.apply('clips16-8',
'cropped16-8',
lambda v: np.asarray([[crop_center(frame) for frame in clip]
for clip in v]),
eval_strategy=mp.EvalStrategies.OnDemand)
# preprocess frames by substracting channel-wise mean
| mp.apply('cropped16-8',
'proc_cropped16-8',
lambda v: preprocess_input(v, mean_std, divide_std=False),
eval_strategy=mp.EvalStrategies.OnDemand)
# run batch predictions on c3d model to get feature vectors for each clip
| mp.apply_batch('proc_cropped16-8',
'c3d16-8',
lambda x: predict_c3d(x, model),
batch_size=batch_size)
# for each full video take feature vectors for all the extracted clips and average them
| mp.apply('c3d16-8',
'c3d_avg',
lambda v: np.average(v, axis=0),
eval_strategy=mp.EvalStrategies.OnDemand)
# draw silly progress
| mp.silly_progress(elements=max_elements)
# save averaged feature vectors into .npy files
| cachecomputex(
ext, target_ext, lambda x, nfn: np.save(nfn, x['c3d_avg']), lambda x,
nfn: print("Skipping saving 'c3d_avg' {}".format(x['filename'])))
| mp.execute)
def fint(x):
return int(float(x))
@mp.Pipe
def import_fields(seq, field_name):
for x in seq:
try:
for k in x[field_name].keys():
x[k] = x[field_name][k]
yield x
except:
print("WARN: object field is {}".format(x[field_name]))
pass
(mp.get_pascal_annotations(os.path.join(data_dir, "Annotations"))
#| mp.take(100)
| mp.unroll("object")
| mp.apply("filename", "fname",
lambda x: os.path.join(data_dir, "JPEGImages", x))
| import_fields('object')
| mp.apply('bndbox_xmin', 'xmin', fint)
| mp.apply('bndbox_xmax', 'xmax', fint)
| mp.apply('bndbox_ymin', 'ymin', fint)
| mp.apply('bndbox_ymax', 'ymax', fint)
| mp.select_fields(['fname', 'xmin', 'ymin', 'xmax', 'ymax', 'name'])
| mp.write_csv('annotations.csv', write_headers=False))
from moviepy.editor import *
import numpy as np
import itertools
import cv2
import math
import matplotlib.pyplot as plt
import keras
from keras.models import Sequential
from keras.layers import *
from keras.regularizers import l2
# Get list of test videos from matches.json
test_names = (from_json(os.path.join(source_dir, 'matches.json'))
| mp.where(lambda x: 'Test' in x.keys() and int(x['Test']) > 0)
| mp.apply(['Id', 'Half'], 'pattern',
lambda x: "{}_{}_".format(x[0], x[1]))
| mp.select_field('pattern')
| mp.as_list)
data = (mp.get_datastream(data_dir, ext=".resized.mp4")
| datasplit_by_pattern(test_pattern=test_names)
| stratify_sample_tt(shuffle=True)
| summary()
| mp.take(1000)
| mp.iter('filename', lambda x: print("Processing {}".format(x)))
| mp.apply(
'filename', 'aud',
lambda x: np.load(x.replace('.resized.mp4', '.audiofeatures.npy')))
| normalize_npy_value('aud', interval=(-1, 1))
| mp.as_list)
# mPyPl - Monadic Pipeline Library for Python
# http://github.com/shwars/mPyPl
# Simple samples
sys.path.append('z:\\GitWork\mPyPl')
import mPyPl as mp
from pipe import *
from mPyPl.utils.pipeutils import *
range(100) | mp.as_field('n') | mp.apply(
'n', 'n5', lambda x: x % 5) | mp.dict_group_by('n5')
data = range(100) | mp.as_field('n') | mp.apply(
'n', 'class_id', lambda x: x % 5) | mp.datasplit(split_value=0.2)
Tr, Te = data | mp.make_train_test_split()
len(Tr | as_list)
len(Te | as_list)
data = range(100) | mp.as_field('n') | mp.apply('n', 'class_id',
lambda x: x % 5) | pshuffle
data | mp.sample_classes('class_id', 1, classes=range(15)) | as_list
x = mp.get_xmlstream_fromdir('e:\\data\\babylon\\')
# mPyPl - Monadic Pipeline Library for Python
# http://github.com/shwars/mPyPl
# Simple samples
sys.path.insert(0, 'z:\\GitWork\mPyPl')
import mPyPl as mp
from pipe import *
from mPyPl.utils.pipeutils import *
import cv2
import mPyPl.utils.image as mpui
print("Using mPyPl version " + mp.__version__)
range(100) | mp.as_field('n') | mp.apply(
'n', 'n5', lambda x: x % 5) | mp.dict_group_by('n5')
data = range(100) | mp.as_field('n') | mp.apply(
'n', 'class_id', lambda x: x % 5) | mp.datasplit(split_value=0.2)
Tr, Te = data | mp.make_train_test_split()
len(Tr | as_list)
len(Te | as_list)
data = range(100) | mp.as_field('n') | mp.apply('n', 'class_id',
lambda x: x % 5) | pshuffle
data | mp.sample_classes('class_id', 1, classes=range(15)) | as_list
x = mp.get_xmlstream_fromdir('e:\\data\\babylon\\')
# Compute sum of integers from 1 to 100. Should return 5050
import functools as fn
import keras
print(mp.__version__)
train_dir = os.path.join(base_dir, 'training_set')
test_dir = os.path.join(base_dir, 'test_set')
classes = mp.get_classes(train_dir)
# we need to explicitly get classes in order to have the same correspondence of class and int for train and test set
# Show first few images from the training set
seq = (mp.get_datastream(train_dir, classes=classes)
| take(10)
| mp.apply(
'filename', 'image',
lambda fn: mpui.im_resize_pad(cv2.imread(fn), size=(100, 100)))
| mp.select_field('image')
| pexec(fn.partial(mpui.show_images, cols=2)))
transform = keras.preprocessing.image.ImageDataGenerator(
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
scale_transform = keras.preprocessing.image.ImageDataGenerator(rescale=1. /
255)
print("Face Dataset Generator")
print(" + Loading descriptions from {}".format(dir))
def loadjs(fn):
with open(fn) as f:
return json.load(f)
min_size = size / 3 if args.ignore_small else 0
max_faces_no = 2 if args.ignore_multiface else 99999
data = (mp.get_files(dir, ext='.json')
| mp.as_field('filename')
| mp.apply('filename', 'descr', loadjs)
| mp.filter('descr', lambda x: len(x) > 0 and len(x) < max_faces_no)
| mp.unroll('descr')
| mp.filter(
'descr', lambda x: abs(x['faceAttributes']['headPose']['yaw']) < 15
and abs(x['faceAttributes']['headPose']['pitch']) < 15)
| mp.filter(
'descr', lambda x: x['faceLandmarks']['pupilRight']['x'] - x[
'faceLandmarks']['pupilLeft']['x'] > min_size)
| mp.as_list)
print(" + Found {} faces".format(len(data)))
print(" + Storing dataset...")
print(f"Processing {match['Video']} ({match['Id']})\n")
videoFilePath = os.path.join(source_dir,match["Video"])
marksFilePath = os.path.join(source_dir,"Marks.jsonl")
video = VideoFileClip(videoFilePath)
data = []
with open(marksFilePath,'r') as f:
for cnt, line in enumerate(f):
data.append(json.loads(line))
# Get list of times when match halves start
mt = data | mp.filter('eventType', lambda x: "start" in x) | mp.select_field('matchTime') | mp.as_list
mt = convtime(mt[h])
# Shots are fine, added also Goal which are not marked as shots
cuts = data | mp.filter('eventType', shotFilter) | mp.apply('matchTime', 'start', convtime)
# Consider, for negative examples, also Attacks, which are Shots nearby the goal area (or in other places of the field)
# They are also filtered (later), removing those overlapping with shots/goals
attacks = data | mp.filter('eventType', attackFilter) | mp.apply('matchTime', 'start', convtime)
clipTrimTime = float(args.clipTrimTime) # +/- X seconds centered on event time
clipTrimAttackTimeRange = float(args.clipTrimAttackTimeRange) # +/- X seconds centered on event time
clipTrimAttackStart = float(args.clipTrimAttackStart) # when attack clip starts (before event time)
clipTrimAttackEnd = float(args.clipTrimAttackEnd) # when attack clip ends (after event time)
noShotClipDuration = float(args.noShotClipDuration)
noShotClipInterval = float(args.noShotClipInterval)
n=0
print(" + Loading images from {}".format(dir))
data = (
mp.get_files(dir)
| mp.as_field('filename')
| mp.apply_nx('filename','image',lambda x: cv2.imread(x),print_exceptions=False)
| mp.filter('image',lambda x: x is not None)
| mp.as_list)
print(" + Found {} images".format(len(data)))
print(" + Extracting facial landmarks...")
data = (
data
| mp.apply(['filename','image'],'landmarks',detect)
| mp.filter('landmarks',lambda x: x!=[])
| mp.as_list
)
if not args.nocache:
print(" + Saving cache...")
with open(os.path.join(script_dir,'cache.pkl'),'wb') as f:
pickle.dump(cache,f)
def transform(args):
image,f = args
mc_x = (f['mouthLeft']['x']+f['mouthRight']['x'])/2.0
mc_y = (f['mouthLeft']['y'] + f['mouthRight']['y']) / 2.0
tr = cv2.getAffineTransform(np.float32([(f['pupilLeft']['x'],f['pupilLeft']['y']),(f['pupilRight']['x'],f['pupilRight']['y']),(mc_x,mc_y)]),
np.float32(target_triangle))
def plot_flow_descriptor(fd, step=5):
fig, ax = plt.subplots(5, 2)
for i in range(5):
ax[i, 0].plot(fd[i * step, :, 0])
ax[i, 1].plot(fd[i * step, :, 1])
plt.show()
# Plot to see how it works
# fd = get_flow_descriptor(flow)
# plot_flow_descriptor(np.log(fd))
# Get list of test videos from matches.json
test_names = (from_json(os.path.join(source_dir, 'matches.json'))
| mp.where(lambda x: 'Test' in x.keys() and int(x['Test']) > 0)
| mp.apply(['Id', 'Half'], 'pattern',
lambda x: "{}_{}_".format(x[0], x[1]))
| mp.select_field('pattern')
| mp.as_list)
data = (mp.get_datastream(data_dir, ext=".resized.mp4")
| datasplit_by_pattern(test_pattern=test_names)
| stratify_sample_tt(shuffle=True)
| summary()
| mp.iter('filename', lambda x: print("Processing {}".format(x)))
| mp.apply(
'filename',
'dflow',
lambda x: np.load(x.replace('.resized.mp4', '.optflow.npy')),
eval_strategy=mp.EvalStrategies.LazyMemoized)
| mp.apply_npy('dflow', 'flowdescr', get_flow_descriptor)
| mp.delfield('dflow')
fx.write_videofile(nfn)
clip.close()
def load_resize(x):
fn = x['filename']
nfn = fn.replace('.full.mp4','.resized.mp4')
x['filename'] = nfn
if os.path.isfile(nfn):
print("Loading resized {}".format(nfn))
vc = VideoFileClip(nfn)
return vc
else:
print("Resizing {}".format(fn))
vc = VideoFileClip(fn).fx(vfx.resize, width=video_width)
vc.write_videofile(nfn)
return vc
def resize(x):
v = VideoFileClip(x)
vfxc = v.fx(vfx.resize, width=video_width)
return (v, vfxc)
if __name__ == "__main__":
(mp.get_datastream(data_dir,ext=".full.mp4")
| where( lambda f: not os.path.isfile( f['filename'].replace(".full.mp4",".resized.mp4") ) )
| mp.apply('filename','video', resize )
| cachecomputex(".full.mp4",".resized.mp4",resize_save,lambda x,nx: print("Skipping {}".format(x['filename'])))
| execute
)
