def __init__(self,
path,
features="ResNET_PCA512.npy",
split="test",
framerate=2,
chunk_size=240,
receptive_field=80):
self.path = path
self.listGames = getListGames(split)
self.features = features
self.chunk_size = chunk_size
self.receptive_field = receptive_field
self.dict_event = EVENT_DICTIONARY_V2
self.num_action_classes = 17
self.labels_actions = "Labels-v2.json"
self.labels_replays = "Labels-cameras.json"
self.framerate = framerate
#logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
downloader.downloadGames(files=[
self.labels_actions, self.labels_replays, f"1_{self.features}",
f"2_{self.features}"
],
split=[split],
verbose=False)
def __init__(self,
path,
features="ResNET_PCA512.npy",
split="test",
version=1,
framerate=2,
chunk_size=240,
receptive_field=80,
num_detections=45):
self.path = path
#self.listGames = getListGames(split)os.path.join(self.dataset_path + self.datatype)
self.listGames = getListGames(split, task="camera-changes")
# self.listGames = np.load(os.path.join(self.path, split))
self.features = features
self.chunk_size = chunk_size
self.receptive_field = receptive_field
self.framerate = framerate
if version == 1:
self.dict_event = EVENT_DICTIONARY_V1
self.num_classes = 3
self.labels = "Labels.json"
self.K_parameters = K_V1 * framerate
self.num_detections = 5
elif version == 2:
self.dict_change = Camera_Change_DICTIONARY
self.dict_type = Camera_Type_DICTIONARY
self.num_classes_sgementation = 13
self.num_classes_camera_change = 1
self.labels = "Labels-cameras.json"
self.K_parameters = K_V2 * framerate
self.num_detections = num_detections
logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
downloader.downloadGames(
files=[self.labels, f"1_{self.features}", f"2_{self.features}"],
split=[split],
verbose=False,
task="camera-changes",
randomized=True)
def __init__(self,
path,
features="ResNET_PCA512.npy",
split="test",
framerate=2,
chunk_size=240,
receptive_field=80):
self.path = path
self.listGames = getListGames(split)
self.features = features
self.chunk_size = chunk_size
self.receptive_field = receptive_field
self.framerate = framerate
self.dict_event = EVENT_DICTIONARY_V2
self.num_classes = 17
self.labels = "Labels-v2.json"
self.K_parameters = K_V2 * framerate
self.num_detections = 15
self.split = split
logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
if split == "challenge":
downloader.downloadGames(
files=[f"1_{self.features}", f"2_{self.features}"],
split=[split],
verbose=False)
else:
downloader.downloadGames(files=[
self.labels, f"1_{self.features}", f"2_{self.features}"
],
split=[split],
verbose=False)
def __init__(self, rootFolder,
feature="ResNET",
video="LQ",
back_end="TF2",
overwrite=False,
transform="crop",
tmp_HQ_videos=None,
grabber="opencv",
FPS=2.0,
split="all"):
self.rootFolder = rootFolder
self.feature = feature
self.video = video
self.back_end = back_end
self.verbose = True
self.transform = transform
self.overwrite = overwrite
self.grabber = grabber
self.FPS = FPS
self.split = split
self.tmp_HQ_videos = tmp_HQ_videos
if self.tmp_HQ_videos:
self.mySoccerNetDownloader = SoccerNetDownloader(self.rootFolder)
self.mySoccerNetDownloader.password = self.tmp_HQ_videos
if "TF2" in self.back_end:
# create pretrained encoder (here ResNet152, pre-trained on ImageNet)
base_model = keras.applications.resnet.ResNet152(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
# define model with output after polling layer (dim=2048)
self.model = Model(base_model.input,
outputs=[base_model.get_layer("avg_pool").output])
self.model.trainable = False
def __init__(self,
path,
features="ResNET_PCA512.npy",
split=["test"],
version=1,
framerate=2,
chunk_size=240):
self.path = path
self.listGames = getListGames(split)
self.features = features
self.chunk_size = chunk_size
self.framerate = framerate
self.version = version
self.split = split
if version == 1:
self.dict_event = EVENT_DICTIONARY_V1
self.num_classes = 3
self.labels = "Labels.json"
elif version == 2:
self.dict_event = EVENT_DICTIONARY_V2
self.num_classes = 17
self.labels = "Labels-v2.json"
logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
for s in split:
if s == "challenge":
downloader.downloadGames(
files=[f"1_{self.features}", f"2_{self.features}"],
split=[s],
verbose=False)
else:
downloader.downloadGames(files=[
self.labels, f"1_{self.features}", f"2_{self.features}"
],
split=[s],
verbose=False)
def __init__(self,
path,
features="ResNET_PCA512.npy",
split="train",
version=1,
framerate=2,
chunk_size=240,
receptive_field=80,
num_detections=45):
self.path = path
self.listGames = getListGames(split, task="camera-changes")
#self.listGames = np.load(os.path.join(self.path, split))
self.features = features
self.chunk_size = chunk_size
self.receptive_field = receptive_field
self.framerate = framerate
if version == 1:
self.dict_event = EVENT_DICTIONARY_V1
self.num_classes = 3
self.labels = "Labels.json"
self.K_parameters = K_V1 * framerate
self.num_detections = 5
elif version == 2:
self.dict_change = Camera_Change_DICTIONARY
self.dict_type = Camera_Type_DICTIONARY
self.num_classes_sgementation = 13
self.num_classes_camera_change = 1
self.labels = "Labels-cameras.json"
self.K_parameters = K_V2 * framerate
self.num_detections = num_detections
logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
downloader.downloadGames(
files=[self.labels, f"1_{self.features}", f"2_{self.features}"],
split=[split],
verbose=False,
task="camera-changes",
randomized=True)
logging.info("Pre-compute clips")
clip_feats = []
clip_labels = []
self.game_feats = list()
self.game_labels = list()
self.game_change_labels = list()
self.game_anchors = list()
game_counter = 0
for game in tqdm(self.listGames):
# Load features
#10% of dataset
# if np.random.randint(10, size=1)<8:
# continue
feat_half1 = np.load(
os.path.join(self.path, game, "1_" + self.features))
feat_half2 = np.load(
os.path.join(self.path, game, "2_" + self.features))
# Load labels
#path2="/content/drive/My Drive/project/path/to/soccernet/data (1)/"
labels = json.load(open(os.path.join(path, game, self.labels)))
label_half1 = np.zeros(
(feat_half1.shape[0], self.num_classes_sgementation))
label_half2 = np.zeros(
(feat_half2.shape[0], self.num_classes_sgementation))
label_change_half1 = np.zeros(
(feat_half1.shape[0], self.num_classes_camera_change))
label_change_half2 = np.zeros(
(feat_half2.shape[0], self.num_classes_camera_change))
for annotation in labels["annotations"]:
time = annotation["gameTime"]
camera_type = annotation["label"]
camera_change = annotation["change_type"]
half = int(time[0])
minutes = int(time[-5:-3])
seconds = int(time[-2::])
framerate = self.framerate
frame = framerate * (seconds + 60 * minutes)
if camera_type in self.dict_type:
label_type = self.dict_type[camera_type]
if half == 1:
frame = min(frame, feat_half1.shape[0] - 1)
label_half1[frame][label_type] = 1
if half == 2:
frame = min(frame, feat_half2.shape[0] - 1)
label_half2[frame][label_type] = 1
#Onehot for camera changee
if camera_change in self.dict_change:
label_change = self.dict_change[camera_change]
if half == 1:
frame = min(frame, feat_half1.shape[0] - 1)
label_change_half1[frame] = 1
if half == 2:
frame = min(frame, feat_half2.shape[0] - 1)
label_change_half2[frame] = 1
shift_half1 = oneHotToAlllabels(label_half1)
shift_half2 = oneHotToAlllabels(label_half2)
#anchors_half1 = getChunks_anchors(shift_half1, game_counter, self.chunk_size, self.receptive_field)
anchors_half1 = getChunks_anchors(label_change_half1, game_counter,
self.chunk_size,
self.receptive_field)
game_counter = game_counter + 1
# with np.printoptions(threshold=np.inf):
# print('anchors_half1',anchors_half1)
#anchors_half2 = getChunks_anchors(shift_half2, game_counter, self.chunk_size, self.receptive_field)
anchors_half2 = getChunks_anchors(label_change_half2, game_counter,
self.chunk_size,
self.receptive_field)
game_counter = game_counter + 1
self.game_feats.append(feat_half1)
self.game_feats.append(feat_half2)
self.game_labels.append(shift_half1)
self.game_labels.append(shift_half2)
# with np.printoptions(threshold=np.inf):
# print('game_labels',self.game_labels)
self.game_change_labels.append(label_change_half1)
self.game_change_labels.append(label_change_half2)
# with np.printoptions(threshold=np.inf):
# print('game_change_labels',self.game_change_labels)
for anchor in anchors_half1:
self.game_anchors.append(anchor)
for anchor in anchors_half2:
self.game_anchors.append(anchor)
import SoccerNet
from SoccerNet.Downloader import SoccerNetDownloader
mySoccerNetDownloader = SoccerNetDownloader(
LocalDirectory="/path/to/SoccerNet")
mySoccerNetDownloader.password = input("Password for videos?:\n")
mySoccerNetDownloader.downloadGames(
files=["Labels-v2.json", "1_ResNET_TF2.npy", "2_ResNET_TF2.npy"],
split=["train", "valid", "test", "challenge"]) # download Features
def __init__(self,
path,
features="ResNET_PCA512.npy",
split=["train"],
version=1,
framerate=2,
chunk_size=240):
self.path = path
self.listGames = getListGames(split)
self.features = features
self.chunk_size = chunk_size
self.version = version
if version == 1:
self.num_classes = 3
self.labels = "Labels.json"
elif version == 2:
self.dict_event = EVENT_DICTIONARY_V2
self.num_classes = 17
self.labels = "Labels-v2.json"
logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
downloader.downloadGames(
files=[self.labels, f"1_{self.features}", f"2_{self.features}"],
split=split,
verbose=False)
logging.info("Pre-compute clips")
self.game_feats = list()
self.game_labels = list()
# game_counter = 0
for game in tqdm(self.listGames):
# Load features
feat_half1 = np.load(
os.path.join(self.path, game, "1_" + self.features))
feat_half1 = feat_half1.reshape(-1, feat_half1.shape[-1])
feat_half2 = np.load(
os.path.join(self.path, game, "2_" + self.features))
feat_half2 = feat_half2.reshape(-1, feat_half2.shape[-1])
# print("feat_half1.shape",feat_half1.shape)
feat_half1 = feats2clip(torch.from_numpy(feat_half1),
stride=self.chunk_size,
clip_length=self.chunk_size)
feat_half2 = feats2clip(torch.from_numpy(feat_half2),
stride=self.chunk_size,
clip_length=self.chunk_size)
# print("feat_half1.shape",feat_half1.shape)
# Load labels
labels = json.load(open(os.path.join(self.path, game,
self.labels)))
label_half1 = np.zeros((feat_half1.shape[0], self.num_classes + 1))
label_half1[:, 0] = 1 # those are BG classes
label_half2 = np.zeros((feat_half2.shape[0], self.num_classes + 1))
label_half2[:, 0] = 1 # those are BG classes
for annotation in labels["annotations"]:
time = annotation["gameTime"]
event = annotation["label"]
half = int(time[0])
minutes = int(time[-5:-3])
seconds = int(time[-2::])
frame = framerate * (seconds + 60 * minutes)
if version == 1:
if "card" in event: label = 0
elif "subs" in event: label = 1
elif "soccer" in event: label = 2
else: continue
elif version == 2:
if event not in self.dict_event:
continue
label = self.dict_event[event]
# if label outside temporal of view
if half == 1 and frame // self.chunk_size >= label_half1.shape[
0]:
continue
if half == 2 and frame // self.chunk_size >= label_half2.shape[
0]:
continue
if half == 1:
label_half1[frame //
self.chunk_size][0] = 0 # not BG anymore
label_half1[frame //
self.chunk_size][label +
1] = 1 # that's my class
if half == 2:
label_half2[frame //
self.chunk_size][0] = 0 # not BG anymore
label_half2[frame //
self.chunk_size][label +
1] = 1 # that's my class
self.game_feats.append(feat_half1)
self.game_feats.append(feat_half2)
self.game_labels.append(label_half1)
self.game_labels.append(label_half2)
self.game_feats = np.concatenate(self.game_feats)
self.game_labels = np.concatenate(self.game_labels)
def __init__(self,
path,
features="ResNET_PCA512.npy",
split="train",
framerate=2,
chunk_size=240,
receptive_field=80,
chunks_per_epoch=6000):
self.path = path
self.listGames = getListGames(split)
self.features = features
self.chunk_size = chunk_size
self.receptive_field = receptive_field
self.chunks_per_epoch = chunks_per_epoch
self.dict_event = EVENT_DICTIONARY_V2
self.num_action_classes = 17
self.labels_actions = "Labels-v2.json"
self.labels_replays = "Labels-cameras.json"
#logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
downloader.downloadGames(files=[
self.labels_actions, self.labels_replays, f"1_{self.features}",
f"2_{self.features}"
],
split=[split],
verbose=False)
logging.info("Pre-compute clips")
clip_feats = []
clip_labels = []
self.game_feats = list()
self.replay_labels = list()
self.replay_anchors = list()
self.game_anchors = list()
game_counter = 0
for game in tqdm(self.listGames):
# Load the features
feat_half1 = np.load(
os.path.join(self.path, game, "1_" + self.features))
feat_half2 = np.load(
os.path.join(self.path, game, "2_" + self.features))
# load the replay labels
labels_replays = json.load(
open(os.path.join(self.path, game, self.labels_replays)))
previous_timestamp = 0
anchors_replay_half1 = list()
anchors_replay_half2 = list()
for annotation in labels_replays["annotations"]:
time = annotation["gameTime"]
half = int(time[0])
minutes = int(time[-5:-3])
seconds = int(time[-2::])
frame = framerate * (seconds + 60 * minutes)
if not "link" in annotation:
previous_timestamp = frame
continue
event = annotation["link"]["label"]
if not event in self.dict_event or int(
annotation["link"]["half"]) != half:
previous_timestamp = frame
continue
if previous_timestamp == frame:
previous_timestamp = frame
continue
time_event = annotation["link"]["time"]
minutes_event = int(time_event[0:2])
seconds_event = int(time_event[3:])
frame_event = framerate * (seconds_event + 60 * minutes_event)
label = self.dict_event[event]
if half == 1:
anchors_replay_half1.append([
game_counter, previous_timestamp, frame, frame_event,
label
])
if half == 2:
anchors_replay_half2.append([
game_counter + 1, previous_timestamp, frame,
frame_event, label
])
previous_timestamp = frame
# Load action labels
labels_actions = json.load(
open(os.path.join(self.path, game, self.labels_actions)))
anchors_half1 = list()
anchors_half2 = list()
for annotation in labels_actions["annotations"]:
time = annotation["gameTime"]
event = annotation["label"]
half = int(time[0])
minutes = int(time[-5:-3])
seconds = int(time[-2::])
frame = framerate * (seconds + 60 * minutes)
if event not in self.dict_event:
continue
label = self.dict_event[event]
if half == 1:
frame = min(frame, feat_half1.shape[0] - 1)
anchors_half1.append([game_counter, frame, label])
if half == 2:
frame = min(frame, feat_half2.shape[0] - 1)
anchors_half2.append([game_counter + 1, frame, label])
self.game_feats.append(feat_half1)
self.game_feats.append(feat_half2)
self.game_anchors.append(list())
for i in np.arange(self.num_action_classes):
self.game_anchors[-1].append(list())
for anchor in anchors_half1:
self.game_anchors[-1][anchor[2]].append(anchor)
self.game_anchors.append(list())
for i in np.arange(self.num_action_classes):
self.game_anchors[-1].append(list())
for anchor in anchors_half2:
self.game_anchors[-1][anchor[2]].append(anchor)
for anchor in anchors_replay_half1:
self.replay_anchors.append(anchor)
for anchor in anchors_replay_half2:
self.replay_anchors.append(anchor)
game_counter = game_counter + 2
def __init__(self,
path,
features="ResNET_PCA512.npy",
split="train",
framerate=2,
chunk_size=240,
receptive_field=80,
chunks_per_epoch=6000):
self.path = path
self.listGames = getListGames(split)
self.features = features
self.chunk_size = chunk_size
self.receptive_field = receptive_field
self.chunks_per_epoch = chunks_per_epoch
self.dict_event = EVENT_DICTIONARY_V2
self.num_classes = 17
self.labels = "Labels-v2.json"
self.K_parameters = K_V2 * framerate
self.num_detections = 15
self.split = split
logging.info("Checking/Download features and labels locally")
downloader = SoccerNetDownloader(path)
downloader.downloadGames(
files=[self.labels, f"1_{self.features}", f"2_{self.features}"],
split=[split],
verbose=False)
logging.info("Pre-compute clips")
clip_feats = []
clip_labels = []
self.game_feats = list()
self.game_labels = list()
self.game_anchors = list()
for i in np.arange(self.num_classes + 1):
self.game_anchors.append(list())
game_counter = 0
for game in tqdm(self.listGames):
# Load features
feat_half1 = np.load(
os.path.join(self.path, game, "1_" + self.features))
feat_half2 = np.load(
os.path.join(self.path, game, "2_" + self.features))
# Load labels
labels = json.load(open(os.path.join(self.path, game,
self.labels)))
label_half1 = np.zeros((feat_half1.shape[0], self.num_classes))
label_half2 = np.zeros((feat_half2.shape[0], self.num_classes))
for annotation in labels["annotations"]:
time = annotation["gameTime"]
event = annotation["label"]
half = int(time[0])
minutes = int(time[-5:-3])
seconds = int(time[-2::])
frame = framerate * (seconds + 60 * minutes)
if event not in self.dict_event:
continue
label = self.dict_event[event]
if half == 1:
frame = min(frame, feat_half1.shape[0] - 1)
label_half1[frame][label] = 1
if half == 2:
frame = min(frame, feat_half2.shape[0] - 1)
label_half2[frame][label] = 1
shift_half1 = oneHotToShifts(label_half1,
self.K_parameters.cpu().numpy())
shift_half2 = oneHotToShifts(label_half2,
self.K_parameters.cpu().numpy())
anchors_half1 = getChunks_anchors(shift_half1, game_counter,
self.K_parameters.cpu().numpy(),
self.chunk_size,
self.receptive_field)
game_counter = game_counter + 1
anchors_half2 = getChunks_anchors(shift_half2, game_counter,
self.K_parameters.cpu().numpy(),
self.chunk_size,
self.receptive_field)
game_counter = game_counter + 1
self.game_feats.append(feat_half1)
self.game_feats.append(feat_half2)
self.game_labels.append(shift_half1)
self.game_labels.append(shift_half2)
for anchor in anchors_half1:
self.game_anchors[anchor[2]].append(anchor)
for anchor in anchors_half2:
self.game_anchors[anchor[2]].append(anchor)
class FeatureExtractor():
def __init__(self, rootFolder,
feature="ResNET",
video="LQ",
back_end="TF2",
overwrite=False,
transform="crop",
tmp_HQ_videos=None,
grabber="opencv",
FPS=2.0,
split="all"):
self.rootFolder = rootFolder
self.feature = feature
self.video = video
self.back_end = back_end
self.verbose = True
self.transform = transform
self.overwrite = overwrite
self.grabber = grabber
self.FPS = FPS
self.split = split
self.tmp_HQ_videos = tmp_HQ_videos
if self.tmp_HQ_videos:
self.mySoccerNetDownloader = SoccerNetDownloader(self.rootFolder)
self.mySoccerNetDownloader.password = self.tmp_HQ_videos
if "TF2" in self.back_end:
# create pretrained encoder (here ResNet152, pre-trained on ImageNet)
base_model = keras.applications.resnet.ResNet152(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
# define model with output after polling layer (dim=2048)
self.model = Model(base_model.input,
outputs=[base_model.get_layer("avg_pool").output])
self.model.trainable = False
def extractAllGames(self):
list_game = getListGames(self.split)
for i_game, game in enumerate(tqdm(list_game)):
try:
self.extractGameIndex(i_game)
except:
print(f"issue with game {i_game}, {game}")
def extractGameIndex(self, index):
print(getListGames(self.split)[index])
if self.video =="LQ":
for vid in ["1.mkv","2.mkv"]:
self.extract(video_path=os.path.join(self.rootFolder, getListGames(self.split)[index], vid))
elif self.video == "HQ":
# read config for raw HD video
config = configparser.ConfigParser()
if not os.path.exists(os.path.join(self.rootFolder, getListGames(self.split)[index], "video.ini")) and self.tmp_HQ_videos is not None:
self.mySoccerNetDownloader.downloadVideoHD(
game=getListGames(self.split)[index], file="video.ini")
config.read(os.path.join(self.rootFolder, getListGames(self.split)[index], "video.ini"))
# lopp over videos
for vid in config.sections():
video_path = os.path.join(self.rootFolder, getListGames(self.split)[index], vid)
# cehck if already exists, then skip
feature_path = video_path[:-4] + f"_{self.feature}_{self.back_end}.npy"
if os.path.exists(feature_path) and not self.overwrite:
print("already exists, early skip")
continue
#Download video if does not exist, but remove it afterwards
remove_afterwards = False
if not os.path.exists(video_path) and self.tmp_HQ_videos is not None:
remove_afterwards = True
self.mySoccerNetDownloader.downloadVideoHD(game=getListGames(self.split)[index], file=vid)
# extract feature for video
self.extract(video_path=video_path,
start=float(config[vid]["start_time_second"]),
duration=float(config[vid]["duration_second"]))
# remove video if not present before
if remove_afterwards:
os.remove(video_path)
def extract(self, video_path, start=None, duration=None):
print("extract video", video_path, "from", start, duration)
# feature_path = video_path.replace(
# ".mkv", f"_{self.feature}_{self.back_end}.npy")
feature_path = video_path[:-4] + f"_{self.feature}_{self.back_end}.npy"
if os.path.exists(feature_path) and not self.overwrite:
return
if "TF2" in self.back_end:
if self.grabber=="skvideo":
videoLoader = Frame(video_path, FPS=self.FPS, transform=self.transform, start=start, duration=duration)
elif self.grabber=="opencv":
videoLoader = FrameCV(video_path, FPS=self.FPS, transform=self.transform, start=start, duration=duration)
# create numpy aray (nb_frames x 224 x 224 x 3)
# frames = np.array(videoLoader.frames)
# if self.preprocess:
frames = preprocess_input(videoLoader.frames)
if duration is None:
duration = videoLoader.time_second
# time_second = duration
if self.verbose:
print("frames", frames.shape, "fps=", frames.shape[0]/duration)
# predict the featrues from the frames (adjust batch size for smalled GPU)
features = self.model.predict(frames, batch_size=64, verbose=1)
if self.verbose:
print("features", features.shape, "fps=", features.shape[0]/duration)
# save the featrue in .npy format
os.makedirs(os.path.dirname(feature_path), exist_ok=True)
np.save(feature_path, features)