def main():
"""
This example shows the use of Statsbomb datasets, and how we can pass argument
to the dataset loader.
"""
logging.basicConfig(
stream=sys.stdout,
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)
dataset = datasets.load(
"statsbomb", {"event_types": ["pass", "take_on", "carry", "shot"]})
with performance_logging("transform", logger=logger):
# convert to TRACAB coordinates
dataset = transform(
dataset,
to_orientation="FIXED_HOME_AWAY",
to_pitch_dimensions=[(-5500, 5500), (-3300, 3300)],
)
with performance_logging("to pandas", logger=logger):
dataframe = to_pandas(dataset)
print(dataframe[:50].to_string())
def test_read(self):
base_dir = os.path.dirname(__file__)
with open(f"{base_dir}/files/epts_meta.xml", "rb") as metadata_fp:
metadata = load_metadata(metadata_fp)
with open(f"{base_dir}/files/epts_raw.txt", "rb") as raw_data:
iterator = read_raw_data(raw_data, metadata)
with performance_logging("load"):
assert list(iterator)
def main():
logging.basicConfig(
stream=sys.stdout,
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
dataset = datasets.load(
"statsbomb", {"event_types": ["pass", "take_on", "carry", "shot"]})
with performance_logging("transform", logger=logger):
dataset = transform(dataset,
to_orientation="FIXED_HOME_AWAY",
to_pitch_dimensions=[(-5500, 5500), (-3300, 3300)])
with performance_logging("to pandas", logger=logger):
dataframe = to_pandas(dataset)
print(dataframe[:50].to_string())
def deserialize(self,
inputs: MetricaEPTSTrackingDataInputs) -> TrackingDataset:
with performance_logging("Loading metadata", logger=logger):
metadata = load_metadata(inputs.meta_data)
if metadata.provider and metadata.pitch_dimensions:
transformer = self.get_transformer(
length=metadata.pitch_dimensions.length,
width=metadata.pitch_dimensions.width,
provider=metadata.coordinate_system.provider,
)
else:
transformer = None
with performance_logging("Loading data", logger=logger):
# assume they are sorted
frames = [
self._frame_from_row(row, metadata, transformer)
for row in read_raw_data(
raw_data=inputs.raw_data,
metadata=metadata,
sensor_ids=[
sensor.sensor_id for sensor in metadata.sensors
],
sample_rate=self.sample_rate,
limit=self.limit,
)
]
if transformer:
metadata = replace(
metadata,
pitch_dimensions=transformer.get_to_coordinate_system().
pitch_dimensions,
coordinate_system=transformer.get_to_coordinate_system(),
)
return TrackingDataset(records=frames, metadata=metadata)
def test_sequence_state_builder(self):
dataset = self._load_dataset()
with performance_logging("add_state"):
dataset_with_state = dataset.add_state("sequence")
events_per_sequence = {}
for sequence_id, events in groupby(
dataset_with_state.events,
lambda event: event.state["sequence"].sequence_id,
):
events = list(events)
events_per_sequence[sequence_id] = len(events)
assert events_per_sequence[0] == 4
assert events_per_sequence[51] == 14
def test_score_state_builder(self):
dataset = self._load_dataset()
with performance_logging("add_state"):
dataset_with_state = dataset.add_state("score")
events_per_score = {}
for score, events in groupby(dataset_with_state.events,
lambda event: event.state["score"]):
events = list(events)
events_per_score[str(score)] = len(events)
assert events_per_score == {
"0-0": 2898,
"1-0": 717,
"2-0": 405,
"3-0": 3,
}
def main():
import pandas as pd
pd.set_option("display.max_colwidth", None)
pd.set_option("display.max_columns", None)
pd.set_option("display.max_rows", None)
pd.set_option("display.width", 2000)
matcher = Matcher(encoder)
dataset = datasets.load("statsbomb",
options={"event_types": ["shot", "pass"]})
with performance_logging("search"):
matches = matcher.search(dataset, r"PPS")
df = to_pandas(
dataset,
additional_columns={
"player_name": lambda event: event.player.full_name,
"team_name": lambda event: str(event.team),
},
)
print(df[["timestamp", "team_name", "player_name", "event_type",
"result"]][:100])
return
for i, match in enumerate(matches):
df = to_pandas(
dataset,
additional_columns={
"player_name": lambda event: event.player.full_name,
"team_name": lambda event: str(event.team),
},
)
print(df[[
"period_id",
"timestamp",
"team_name",
"player_name",
"event_type",
]])
def test_register_custom_builder(self):
class CustomStateBuilder(StateBuilder):
def initial_state(self, dataset: EventDataset) -> int:
return 0
def reduce_before(self, state: int, event: Event) -> int:
return state + 1
def reduce_after(self, state: int, event: Event) -> int:
return state + 1
dataset = self._load_dataset("statsbomb_15986")
with performance_logging("add_state"):
dataset_with_state = dataset.add_state("custom")
assert dataset_with_state.events[0].state["custom"] == 1
assert dataset_with_state.events[1].state["custom"] == 3
assert dataset_with_state.events[2].state["custom"] == 5
assert dataset_with_state.events[3].state["custom"] == 7
def test_formation_state_builder(self):
dataset = self._load_dataset("statsbomb")
with performance_logging("add_state"):
dataset_with_state = dataset.add_state("formation")
events_per_formation_change = {}
for formation, events in groupby(
dataset_with_state.events,
lambda event: event.state["formation"].away,
):
events = list(events)
events_per_formation_change[str(formation)] = len(events)
# inspect FormationChangeEvent usage and formation state_builder
assert events_per_formation_change["4-1-4-1"] == 3074
assert events_per_formation_change["4-4-2"] == 949
assert dataset.metadata.teams[0].starting_formation == FormationType(
"4-4-2")
assert dataset_with_state.events[1].state[
"formation"].home == FormationType("4-4-2")
def test_lineup_state_builder(self):
dataset = self._load_dataset("statsbomb_15986")
with performance_logging("add_state"):
dataset_with_state = dataset.add_state("lineup")
last_events = []
for lineup, events in groupby(dataset_with_state.events,
lambda event: event.state["lineup"]):
events = list(events)
# inspect last event which changed the lineup
last_events.append((events[-1].event_type, len(lineup.players)))
assert last_events == [
(EventType.CARD, 22),
(EventType.SUBSTITUTION, 21),
(EventType.SUBSTITUTION, 21),
(EventType.SUBSTITUTION, 21),
(EventType.SUBSTITUTION, 21),
(EventType.SUBSTITUTION, 21),
(EventType.SUBSTITUTION, 21),
(EventType.GENERIC, 21),
]
def deserialize(self, inputs: SecondSpectrumInputs) -> TrackingDataset:
metadata = None
# Handles the XML metadata that contains the pitch dimensions and frame info
with performance_logging("Loading XML metadata", logger=logger):
# The meta data can also be in JSON format. In that case
# it also contains the 'additional metadata'.
# First do a 'peek' to determine the char
first_byte = inputs.meta_data.read(1)
if first_byte == b"{":
metadata = json.loads(first_byte + inputs.meta_data.read())
frame_rate = int(metadata["fps"])
pitch_size_height = float(metadata["pitchLength"])
pitch_size_width = float(metadata["pitchWidth"])
periods = []
for period in metadata["periods"]:
start_frame_id = int(period["startFrameIdx"])
end_frame_id = int(period["endFrameIdx"])
if start_frame_id != 0 or end_frame_id != 0:
# Frame IDs are unix timestamps (in milliseconds)
periods.append(
Period(
id=int(period["number"]),
start_timestamp=start_frame_id,
end_timestamp=end_frame_id,
)
)
else:
match = objectify.fromstring(
first_byte + inputs.meta_data.read()
).match
frame_rate = int(match.attrib["iFrameRateFps"])
pitch_size_height = float(match.attrib["fPitchYSizeMeters"])
pitch_size_width = float(match.attrib["fPitchXSizeMeters"])
periods = []
for period in match.iterchildren(tag="period"):
start_frame_id = int(period.attrib["iStartFrame"])
end_frame_id = int(period.attrib["iEndFrame"])
if start_frame_id != 0 or end_frame_id != 0:
# Frame IDs are unix timestamps (in milliseconds)
periods.append(
Period(
id=int(period.attrib["iId"]),
start_timestamp=start_frame_id,
end_timestamp=end_frame_id,
)
)
# Default team initialisation
home_team = Team(team_id="home", name="home", ground=Ground.HOME)
away_team = Team(team_id="away", name="away", ground=Ground.AWAY)
teams = [home_team, away_team]
if inputs.additional_meta_data or metadata:
with performance_logging("Loading JSON metadata", logger=logger):
try:
if inputs.additional_meta_data:
metadata = json.loads(
inputs.additional_meta_data.read()
)
home_team_id = metadata["homeOptaId"]
away_team_id = metadata["awayOptaId"]
# Tries to parse (short) team names from the description string
try:
home_name = (
metadata["description"].split("-")[0].strip()
)
away_name = (
metadata["description"]
.split("-")[1]
.split(":")[0]
.strip()
)
except:
home_name, away_name = "home", "away"
teams[0].team_id = home_team_id
teams[0].name = home_name
teams[1].team_id = away_team_id
teams[1].name = away_name
for team, team_str in zip(
teams, ["homePlayers", "awayPlayers"]
):
for player_data in metadata[team_str]:
# We use the attributes field of Player to store the extra IDs provided by the
# metadata. We designate the player_id to be the 'optaId' field as this is what's
# used as 'player_id' in the raw frame data file
player_attributes = {
k: v
for k, v in player_data.items()
if k in ["ssiId", "optaUuid"]
}
player = Player(
player_id=player_data["optaId"],
name=player_data["name"],
starting=player_data["position"] != "SUB",
position=player_data["position"],
team=team,
jersey_no=int(player_data["number"]),
attributes=player_attributes,
)
team.players.append(player)
except: # TODO: More specific exception
logging.warning(
"Optional JSON Metadata is malformed. Continuing without"
)
# Handles the tracking frame data
with performance_logging("Loading data", logger=logger):
transformer = self.get_transformer(
length=pitch_size_width, width=pitch_size_height
)
def _iter():
n = 0
sample = 1 / self.sample_rate
for line_ in inputs.raw_data.readlines():
line_ = line_.strip().decode("ascii")
if not line_:
continue
# Each line is just json so we just parse it
frame_data = json.loads(line_)
if self.only_alive and not frame_data["live"]:
continue
if n % sample == 0:
yield frame_data
n += 1
frames = []
for n, frame_data in enumerate(_iter()):
period = periods[frame_data["period"] - 1]
frame = self._frame_from_framedata(teams, period, frame_data)
frame = transformer.transform_frame(frame)
frames.append(frame)
if not period.attacking_direction_set:
period.set_attacking_direction(
attacking_direction=attacking_direction_from_frame(
frame
)
)
if self.limit and n + 1 >= self.limit:
break
orientation = (
Orientation.FIXED_HOME_AWAY
if periods[0].attacking_direction == AttackingDirection.HOME_AWAY
else Orientation.FIXED_AWAY_HOME
)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=transformer.get_to_coordinate_system().pitch_dimensions,
score=None,
frame_rate=frame_rate,
orientation=orientation,
provider=Provider.SECONDSPECTRUM,
flags=DatasetFlag.BALL_OWNING_TEAM | DatasetFlag.BALL_STATE,
coordinate_system=transformer.get_to_coordinate_system(),
)
return TrackingDataset(
records=frames,
metadata=metadata,
)
def deserialize(self, inputs: OptaInputs) -> EventDataset:
transformer = self.get_transformer(length=100, width=100)
with performance_logging("load data", logger=logger):
f7_root = objectify.fromstring(inputs.f7_data.read())
f24_root = objectify.fromstring(inputs.f24_data.read())
with performance_logging("parse data", logger=logger):
matchdata_path = objectify.ObjectPath(
"SoccerFeed.SoccerDocument.MatchData")
team_elms = list(
matchdata_path.find(f7_root).iterchildren("TeamData"))
home_score = None
away_score = None
for team_elm in team_elms:
if team_elm.attrib["Side"] == "Home":
home_score = team_elm.attrib["Score"]
home_team = _team_from_xml_elm(team_elm, f7_root)
elif team_elm.attrib["Side"] == "Away":
away_score = team_elm.attrib["Score"]
away_team = _team_from_xml_elm(team_elm, f7_root)
else:
raise DeserializationError(
f"Unknown side: {team_elm.attrib['Side']}")
score = Score(home=home_score, away=away_score)
teams = [home_team, away_team]
if len(home_team.players) == 0 or len(away_team.players) == 0:
raise DeserializationError("LineUp incomplete")
game_elm = f24_root.find("Game")
periods = [
Period(
id=1,
start_timestamp=None,
end_timestamp=None,
),
Period(
id=2,
start_timestamp=None,
end_timestamp=None,
),
]
possession_team = None
events = []
for event_elm in game_elm.iterchildren("Event"):
event_id = event_elm.attrib["id"]
type_id = int(event_elm.attrib["type_id"])
timestamp = _parse_f24_datetime(event_elm.attrib["timestamp"])
period_id = int(event_elm.attrib["period_id"])
for period in periods:
if period.id == period_id:
break
else:
logger.debug(
f"Skipping event {event_id} because period doesn't match {period_id}"
)
continue
if type_id == EVENT_TYPE_START_PERIOD:
logger.debug(
f"Set start of period {period.id} to {timestamp}")
period.start_timestamp = timestamp
elif type_id == EVENT_TYPE_END_PERIOD:
logger.debug(
f"Set end of period {period.id} to {timestamp}")
period.end_timestamp = timestamp
else:
if not period.start_timestamp:
# not started yet
continue
if event_elm.attrib["team_id"] == home_team.team_id:
team = teams[0]
elif event_elm.attrib["team_id"] == away_team.team_id:
team = teams[1]
else:
raise DeserializationError(
f"Unknown team_id {event_elm.attrib['team_id']}")
x = float(event_elm.attrib["x"])
y = float(event_elm.attrib["y"])
outcome = int(event_elm.attrib["outcome"])
raw_qualifiers = {
int(qualifier_elm.attrib["qualifier_id"]):
qualifier_elm.attrib.get("value")
for qualifier_elm in event_elm.iterchildren("Q")
}
player = None
if "player_id" in event_elm.attrib:
player = team.get_player_by_id(
event_elm.attrib["player_id"])
if type_id in BALL_OWNING_EVENTS:
possession_team = team
generic_event_kwargs = dict(
# from DataRecord
period=period,
timestamp=timestamp - period.start_timestamp,
ball_owning_team=possession_team,
ball_state=BallState.ALIVE,
# from Event
event_id=event_id,
team=team,
player=player,
coordinates=Point(x=x, y=y),
raw_event=event_elm,
)
if type_id == EVENT_TYPE_PASS:
pass_event_kwargs = _parse_pass(
raw_qualifiers, outcome)
event = PassEvent.create(
**pass_event_kwargs,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_OFFSIDE_PASS:
pass_event_kwargs = _parse_offside_pass(raw_qualifiers)
event = PassEvent.create(
**pass_event_kwargs,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_TAKE_ON:
take_on_event_kwargs = _parse_take_on(outcome)
event = TakeOnEvent.create(
qualifiers=None,
**take_on_event_kwargs,
**generic_event_kwargs,
)
elif type_id in (
EVENT_TYPE_SHOT_MISS,
EVENT_TYPE_SHOT_POST,
EVENT_TYPE_SHOT_SAVED,
EVENT_TYPE_SHOT_GOAL,
):
if type_id == EVENT_TYPE_SHOT_GOAL:
if 374 in raw_qualifiers.keys():
generic_event_kwargs["timestamp"] = (
_parse_f24_datetime(
raw_qualifiers.get(374).replace(
" ", "T")) -
period.start_timestamp)
shot_event_kwargs = _parse_shot(
raw_qualifiers,
type_id,
coordinates=generic_event_kwargs["coordinates"],
)
kwargs = {}
kwargs.update(generic_event_kwargs)
kwargs.update(shot_event_kwargs)
event = ShotEvent.create(**kwargs)
elif type_id == EVENT_TYPE_RECOVERY:
event = RecoveryEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_FOUL_COMMITTED:
event = FoulCommittedEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif type_id in BALL_OUT_EVENTS:
generic_event_kwargs["ball_state"] = BallState.DEAD
event = BallOutEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_FORMATION_CHANGE:
formation_change_event_kwargs = (
_parse_formation_change(raw_qualifiers))
event = FormationChangeEvent.create(
result=None,
qualifiers=None,
**formation_change_event_kwargs,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_CARD:
generic_event_kwargs["ball_state"] = BallState.DEAD
card_event_kwargs = _parse_card(raw_qualifiers)
event = CardEvent.create(
**card_event_kwargs,
**generic_event_kwargs,
)
else:
event = GenericEvent.create(
**generic_event_kwargs,
result=None,
qualifiers=None,
event_name=_get_event_type_name(type_id),
)
if self.should_include_event(event):
events.append(transformer.transform_event(event))
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=transformer.get_to_coordinate_system().
pitch_dimensions,
score=score,
frame_rate=None,
orientation=Orientation.ACTION_EXECUTING_TEAM,
flags=DatasetFlag.BALL_OWNING_TEAM,
provider=Provider.OPTA,
coordinate_system=transformer.get_to_coordinate_system(),
)
return EventDataset(
metadata=metadata,
records=events,
)
def deserialize(self,
inputs: Dict[str, Readable],
options: Dict = None) -> TrackingDataset:
"""
Deserialize SkillCorner tracking data into a `TrackingDataset`.
Parameters
----------
inputs : dict
input `raw_data` should point to a `Readable` object containing
the 'json' formatted raw data. input `metadata` should point to
the json metadata data.
options : dict
Options for deserialization of the TRACAB file. Possible options are:
`include_empty_frames` (boolean): default = False to specify whether frames without
any players_coordinates or the ball_coordinates should be loaded
`sample_rate` (float between 0 and 1) to specify the amount of frames that should be loaded
and `limit` (int) to specify the max number of frames that will be returned.
Returns
-------
dataset : TrackingDataset
Raises
------
-
See Also
--------
Examples
--------
>>> serializer = SkillCornerSerializer()
>>> with open("match_data.json", "rb") as meta, \
>>> open("structured_data.json", "rb") as raw:
>>> dataset = serializer.deserialize(
>>> inputs={
>>> 'metadata': meta,
>>> 'raw_data': raw
>>> },
>>> options={
>>> }
>>> )
"""
self.__validate_inputs(inputs)
metadata = self.__load_json(inputs["metadata"])
raw_data = self.__load_json(inputs["raw_data"])
if not options:
options = {}
sample_rate = float(options.get("sample_rate", 1.0))
limit = int(options.get("limit", 0))
include_empty_frames = bool(options.get("include_empty_frames", False))
with performance_logging("Loading metadata", logger=logger):
periods = self.__get_periods(raw_data)
teamdict = {
metadata["home_team"].get("id"): "home_team",
metadata["away_team"].get("id"): "away_team",
}
player_id_to_team_dict = {
player["trackable_object"]: player["team_id"]
for player in metadata["players"]
}
player_dict = {
player["trackable_object"]: player
for player in metadata["players"]
}
referee_dict = {
ref["trackable_object"]: "referee"
for ref in metadata["referees"]
}
ball_id = metadata["ball"]["trackable_object"]
# there are different pitch_sizes in SkillCorner
pitch_size_width = metadata["pitch_width"]
pitch_size_length = metadata["pitch_length"]
home_team_id = metadata["home_team"]["id"]
away_team_id = metadata["away_team"]["id"]
players = {"HOME": {}, "AWAY": {}}
home_team = Team(
team_id=home_team_id,
name=metadata["home_team"]["name"],
ground=Ground.HOME,
)
self.home_team = home_team
away_team = Team(
team_id=away_team_id,
name=metadata["away_team"]["name"],
ground=Ground.AWAY,
)
self.away_team = away_team
teams = [home_team, away_team]
for player_id in player_dict.keys():
player = player_dict.get(player_id)
team_id = player["team_id"]
if team_id == home_team_id:
team_string = "HOME"
team = home_team
elif team_id == away_team_id:
team_string = "AWAY"
team = away_team
players[team_string][player_id] = Player(
player_id=f"{team.ground}_{player['number']}",
team=team,
jersey_no=player["number"],
name=f"{player['first_name']} {player['last_name']}",
first_name=player["first_name"],
last_name=player["last_name"],
starting=player["start_time"] == "00:00:00",
position=Position(
position_id=player["player_role"].get("id"),
name=player["player_role"].get("name"),
coordinates=None,
),
attributes={},
)
home_team.players = list(players["HOME"].values())
away_team.players = list(players["AWAY"].values())
anon_players = {"HOME": {}, "AWAY": {}}
with performance_logging("Loading data", logger=logger):
def _iter():
n = 0
sample = 1.0 / sample_rate
for frame in raw_data:
frame_period = frame["period"]
if frame_period is not None:
if n % sample == 0:
yield frame
n += 1
frames = []
n_frames = 0
for _frame in _iter():
# include frame if there is any tracking data, players or ball.
# or if include_empty_frames == True
if include_empty_frames or len(_frame["data"]) > 0:
frame = self._get_frame_data(
teams,
teamdict,
players,
player_id_to_team_dict,
periods,
player_dict,
anon_players,
ball_id,
referee_dict,
_frame,
)
frames.append(frame)
n_frames += 1
if limit and n_frames >= limit:
break
self._set_skillcorner_attacking_directions(frames, periods)
frame_rate = 10
orientation = (Orientation.HOME_TEAM if periods[1].attacking_direction
== AttackingDirection.HOME_AWAY else
Orientation.AWAY_TEAM)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=PitchDimensions(
x_dim=Dimension(-(pitch_size_length / 2),
(pitch_size_length / 2)),
y_dim=Dimension(-(pitch_size_width / 2),
(pitch_size_width / 2)),
x_per_meter=1,
y_per_meter=1,
),
score=Score(
home=metadata["home_team_score"],
away=metadata["away_team_score"],
),
frame_rate=frame_rate,
orientation=orientation,
provider=Provider.SKILLCORNER,
flags=~(DatasetFlag.BALL_STATE | DatasetFlag.BALL_OWNING_TEAM),
)
return TrackingDataset(
records=frames,
metadata=metadata,
)
def deserialize(self,
inputs: Dict[str, Readable],
options: Dict = None) -> TrackingDataset:
"""
Deserialize Metrica tracking data into a `TrackingDataset`.
Parameters
----------
inputs : dict
input `raw_data_home` should point to a `Readable` object containing
the 'csv' formatted raw data for the home team. input `raw_data_away` should point
to a `Readable` object containing the 'csv' formatted raw data for the away team.
options : dict
Options for deserialization of the Metrica file. Possible options are
`sample_rate` (float between 0 and 1) to specify the amount of
frames that should be loaded, `limit` to specify the max number of
frames that will be returned.
Returns
-------
dataset : TrackingDataset
Raises
------
ValueError when both input files don't seem to belong to each other
See Also
--------
Examples
--------
>>> serializer = MetricaTrackingSerializer()
>>> with open("Sample_Game_1_RawTrackingData_Away_Team.csv", "rb") as raw_home, \
>>> open("Sample_Game_1_RawTrackingData_Home_Team.csv", "rb") as raw_away:
>>>
>>> dataset = serializer.deserialize(
>>> inputs={
>>> 'raw_data_home': raw_home,
>>> 'raw_data_away': raw_away
>>> },
>>> options={
>>> 'sample_rate': 1/12
>>> }
>>> )
"""
self.__validate_inputs(inputs)
if not options:
options = {}
sample_rate = float(options.get("sample_rate", 1.0))
limit = int(options.get("limit", 0))
# consider reading this from data
frame_rate = 25
with performance_logging("prepare", logger=logger):
home_iterator = self.__create_iterator(inputs["raw_data_home"],
sample_rate, frame_rate,
Ground.HOME)
away_iterator = self.__create_iterator(inputs["raw_data_away"],
sample_rate, frame_rate,
Ground.AWAY)
partial_frames = zip(home_iterator, away_iterator)
with performance_logging("loading", logger=logger):
frames = []
periods = []
partial_frame_type = self.__PartialFrame
home_partial_frame: partial_frame_type
away_partial_frame: partial_frame_type
for n, (home_partial_frame,
away_partial_frame) in enumerate(partial_frames):
self.__validate_partials(home_partial_frame,
away_partial_frame)
period: Period = home_partial_frame.period
frame_id: int = home_partial_frame.frame_id
players_coordinates = {
**home_partial_frame.players_coordinates,
**away_partial_frame.players_coordinates,
}
frame = Frame(
frame_id=frame_id,
timestamp=frame_id / frame_rate - period.start_timestamp,
ball_coordinates=home_partial_frame.ball_coordinates,
players_coordinates=players_coordinates,
period=period,
ball_state=None,
ball_owning_team=None,
)
frames.append(frame)
if not periods or period.id != periods[-1].id:
periods.append(period)
if not period.attacking_direction_set:
period.set_attacking_direction(
attacking_direction=attacking_direction_from_frame(
frame))
if n == 0:
teams = [home_partial_frame.team, away_partial_frame.team]
n += 1
if limit and n >= limit:
break
orientation = (
Orientation.FIXED_HOME_AWAY if periods[0].attacking_direction
== AttackingDirection.HOME_AWAY else Orientation.FIXED_AWAY_HOME)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=PitchDimensions(x_dim=Dimension(0, 1),
y_dim=Dimension(0, 1)),
score=None,
frame_rate=frame_rate,
orientation=orientation,
provider=Provider.METRICA,
flags=~(DatasetFlag.BALL_STATE | DatasetFlag.BALL_OWNING_TEAM),
)
return TrackingDataset(records=frames, metadata=metadata)
def deserialize(self,
inputs: Dict[str, Readable],
options: Dict = None) -> TrackingDataset:
"""
Deserialize EPTS tracking data into a `TrackingDataset`.
Parameters
----------
inputs : dict
input `raw_data` should point to a `Readable` object containing
the 'csv' formatted raw data. input `metadata` should point to
the xml metadata data.
options : dict
Options for deserialization of the EPTS file. Possible options are
`sample_rate` (float between 0 and 1) to specify the amount of
frames that should be loaded, `limit` to specify the max number of
frames that will be returned.
Returns
-------
dataset : TrackingDataset
Raises
------
-
See Also
--------
Examples
--------
>>> serializer = EPTSSerializer()
>>> with open("metadata.xml", "rb") as meta, \
>>> open("raw.dat", "rb") as raw:
>>> dataset = serializer.deserialize(
>>> inputs={
>>> 'metadata': meta,
>>> 'raw_data': raw
>>> },
>>> options={
>>> 'sample_rate': 1/12
>>> }
>>> )
"""
self.__validate_inputs(inputs)
if not options:
options = {}
sample_rate = float(options.get("sample_rate", 1.0))
limit = int(options.get("limit", 0))
with performance_logging("Loading metadata", logger=logger):
metadata = load_metadata(inputs["metadata"])
with performance_logging("Loading data", logger=logger):
# assume they are sorted
frames = [
self._frame_from_row(row, metadata) for row in read_raw_data(
raw_data=inputs["raw_data"],
metadata=metadata,
sensor_ids=["position"
], # we don't care about other sensors
sample_rate=sample_rate,
limit=limit,
)
]
return TrackingDataset(records=frames, metadata=metadata)
def deserialize(self,
inputs: Dict[str, Readable],
options: Dict = None) -> EventDataset:
"""
Deserialize StatsBomb event data into a `EventDataset`.
Parameters
----------
inputs : dict
input `event_data` should point to a `Readable` object containing
the 'json' formatted event data. input `lineup_data` should point
to a `Readable` object containing the 'json' formatted lineup data.
options : dict
Options for deserialization of the StatsBomb file. Possible options are
`event_types` (list of event types) to specify the event types that
should be returned. Valid types: "shot", "pass", "carry", "take_on" and
"generic". Generic is everything other than the first 4. Those events
are barely parsed. This type of event can be used to do the parsing
yourself.
Every event has a 'raw_event' attribute which contains the original
dictionary.
Returns
-------
dataset : EventDataset
Raises
------
See Also
--------
Examples
--------
>>> serializer = StatsBombSerializer()
>>> with open("events/12312312.json", "rb") as event_data, \
>>> open("lineups/123123123.json", "rb") as lineup_data:
>>>
>>> dataset = serializer.deserialize(
>>> inputs={
>>> 'event_data': event_data,
>>> 'lineup_data': lineup_data
>>> },
>>> options={
>>> 'event_types': ["pass", "take_on", "carry", "shot"]
>>> }
>>> )
"""
self.__validate_inputs(inputs)
if not options:
options = {}
with performance_logging("load data", logger=logger):
raw_events = json.load(inputs["event_data"])
home_lineup, away_lineup = json.load(inputs["lineup_data"])
(
shot_fidelity_version,
xy_fidelity_version,
) = _determine_xy_fidelity_versions(raw_events)
logger.info(
f"Determined Fidelity versions: shot v{shot_fidelity_version} / XY v{xy_fidelity_version}"
)
with performance_logging("parse data", logger=logger):
starting_player_ids = {
str(player["player"]["id"])
for raw_event in raw_events
if raw_event["type"]["id"] == SB_EVENT_TYPE_STARTING_XI
for player in raw_event["tactics"]["lineup"]
}
home_team = Team(
team_id=str(home_lineup["team_id"]),
name=home_lineup["team_name"],
ground=Ground.HOME,
)
home_team.players = [
Player(
player_id=str(player["player_id"]),
team=home_team,
name=player["player_name"],
jersey_no=int(player["jersey_number"]),
starting=str(player["player_id"]) in starting_player_ids,
) for player in home_lineup["lineup"]
]
away_team = Team(
team_id=str(away_lineup["team_id"]),
name=away_lineup["team_name"],
ground=Ground.AWAY,
)
away_team.players = [
Player(
player_id=str(player["player_id"]),
team=away_team,
name=player["player_name"],
jersey_no=int(player["jersey_number"]),
starting=str(player["player_id"]) in starting_player_ids,
) for player in away_lineup["lineup"]
]
teams = [home_team, away_team]
wanted_event_types = [
EventType[event_type.upper()]
for event_type in options.get("event_types", [])
]
periods = []
period = None
events = []
for raw_event in raw_events:
if raw_event["team"]["id"] == home_lineup["team_id"]:
team = teams[0]
elif raw_event["team"]["id"] == away_lineup["team_id"]:
team = teams[1]
else:
raise Exception(
f"Unknown team_id {raw_event['team']['id']}")
if (raw_event["possession_team"]["id"] ==
home_lineup["team_id"]):
possession_team = teams[0]
elif (raw_event["possession_team"]["id"] ==
away_lineup["team_id"]):
possession_team = teams[1]
else:
raise Exception(
f"Unknown possession_team_id: {raw_event['possession_team']}"
)
timestamp = parse_str_ts(raw_event["timestamp"])
period_id = int(raw_event["period"])
if not period or period.id != period_id:
period = Period(
id=period_id,
start_timestamp=(
timestamp if not period
# period = [start, end], add millisecond to prevent overlapping
else timestamp + period.end_timestamp + 0.001),
end_timestamp=None,
)
periods.append(period)
else:
period.end_timestamp = period.start_timestamp + timestamp
player = None
if "player" in raw_event:
player = team.get_player_by_id(raw_event["player"]["id"])
event_type = raw_event["type"]["id"]
if event_type == SB_EVENT_TYPE_SHOT:
fidelity_version = shot_fidelity_version
elif event_type in (
SB_EVENT_TYPE_CARRY,
SB_EVENT_TYPE_DRIBBLE,
SB_EVENT_TYPE_PASS,
):
fidelity_version = xy_fidelity_version
else:
# TODO: Uh ohhhh.. don't know which one to pick
fidelity_version = xy_fidelity_version
generic_event_kwargs = dict(
# from DataRecord
period=period,
timestamp=timestamp,
ball_owning_team=possession_team,
ball_state=BallState.ALIVE,
# from Event
event_id=raw_event["id"],
team=team,
player=player,
coordinates=(_parse_coordinates(raw_event.get("location"),
fidelity_version)
if "location" in raw_event else None),
raw_event=raw_event,
)
if event_type == SB_EVENT_TYPE_PASS:
pass_event_kwargs = _parse_pass(
pass_dict=raw_event["pass"],
team=team,
fidelity_version=fidelity_version,
)
event = PassEvent.create(
# TODO: Consider moving this to _parse_pass
receive_timestamp=timestamp + raw_event["duration"],
**pass_event_kwargs,
**generic_event_kwargs,
)
elif event_type == SB_EVENT_TYPE_SHOT:
shot_event_kwargs = _parse_shot(
shot_dict=raw_event["shot"])
event = ShotEvent.create(**shot_event_kwargs,
**generic_event_kwargs)
# For dribble and carry the definitions
# are flipped between Statsbomb and kloppy
elif event_type == SB_EVENT_TYPE_DRIBBLE:
take_on_event_kwargs = _parse_take_on(
take_on_dict=raw_event["dribble"])
event = TakeOnEvent.create(
qualifiers=None,
**take_on_event_kwargs,
**generic_event_kwargs,
)
elif event_type == SB_EVENT_TYPE_CARRY:
carry_event_kwargs = _parse_carry(
carry_dict=raw_event["carry"],
fidelity_version=fidelity_version,
)
event = CarryEvent.create(
qualifiers=None,
# TODO: Consider moving this to _parse_carry
end_timestamp=timestamp + raw_event["duration"],
**carry_event_kwargs,
**generic_event_kwargs,
)
# lineup affecting events
elif event_type == SB_EVENT_TYPE_SUBSTITUTION:
substitution_event_kwargs = _parse_substitution(
substitution_dict=raw_event["substitution"], team=team)
event = SubstitutionEvent.create(
result=None,
qualifiers=None,
**substitution_event_kwargs,
**generic_event_kwargs,
)
elif event_type == SB_EVENT_TYPE_BAD_BEHAVIOUR:
card_kwargs = _parse_card(
card_containing_dict=raw_event.get(
"bad_behaviour", {}))
if card_kwargs["card_type"]:
event = CardEvent.create(
result=None,
qualifiers=None,
card_type=card_kwargs["card_type"],
**generic_event_kwargs,
)
elif event_type == SB_EVENT_TYPE_FOUL_COMMITTED:
card_kwargs = _parse_card(
card_containing_dict=raw_event.get(
"foul_committed", {}))
if card_kwargs["card_type"]:
event = CardEvent.create(
result=None,
qualifiers=None,
card_type=card_kwargs["card_type"],
**generic_event_kwargs,
)
elif event_type == SB_EVENT_TYPE_PLAYER_ON:
event = PlayerOnEvent.create(result=None,
qualifiers=None,
**generic_event_kwargs)
elif event_type == SB_EVENT_TYPE_PLAYER_OFF:
event = PlayerOffEvent.create(result=None,
qualifiers=None,
**generic_event_kwargs)
elif event_type == SB_EVENT_TYPE_RECOVERY:
event = RecoveryEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif event_type == SB_EVENT_TYPE_FOUL_COMMITTED:
event = FoulCommittedEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
# rest: generic
else:
event = GenericEvent.create(
result=None,
qualifiers=None,
event_name=raw_event["type"]["name"],
**generic_event_kwargs,
)
if _include_event(event, wanted_event_types):
events.append(event)
# Checks if the event ended out of the field and adds a synthetic out event
if event.result in OUT_EVENT_RESULTS:
generic_event_kwargs["ball_state"] = BallState.DEAD
if event.receiver_coordinates:
generic_event_kwargs[
"coordinates"] = event.receiver_coordinates
event = BallOutEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
if _include_event(event, wanted_event_types):
events.append(event)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=PitchDimensions(x_dim=Dimension(0, 120),
y_dim=Dimension(0, 80)),
frame_rate=None,
orientation=Orientation.ACTION_EXECUTING_TEAM,
flags=DatasetFlag.BALL_OWNING_TEAM,
score=None,
provider=Provider.STATSBOMB,
)
return EventDataset(
metadata=metadata,
records=events,
)
def deserialize(
self, inputs: Dict[str, Readable], options: Dict = None
) -> EventDataset:
WyscoutSerializer.__validate_inputs(inputs)
if not options:
options = {}
wanted_event_types = [
EventType[event_type.upper()]
for event_type in options.get("event_types", [])
]
with performance_logging("load data", logger=logger):
raw_events = json.load(inputs["event_data"])
periods = []
with performance_logging("parse data", logger=logger):
home_team_id, away_team_id = raw_events["teams"].keys()
home_team = _parse_team(raw_events, home_team_id, Ground.HOME)
away_team = _parse_team(raw_events, away_team_id, Ground.AWAY)
teams = {home_team_id: home_team, away_team_id: away_team}
players = dict(
[
(wyId, _players_to_dict(team.players))
for wyId, team in teams.items()
]
)
events = []
for idx, raw_event in enumerate(raw_events["events"]):
next_event = None
if (idx + 1) < len(raw_events["events"]):
next_event = raw_events["events"][idx + 1]
team_id = str(raw_event["teamId"])
player_id = str(raw_event["playerId"])
if (
len(periods) == 0
or periods[-1].id != raw_event["matchPeriod"]
):
periods.append(
Period(
id=raw_event["matchPeriod"],
start_timestamp=0,
end_timestamp=0,
)
)
generic_event_args = {
"event_id": raw_event["id"],
"raw_event": raw_event,
"coordinates": Point(
x=float(raw_event["positions"][0]["x"]),
y=float(raw_event["positions"][0]["y"]),
),
"team": teams[team_id],
"player": players[team_id][player_id]
if player_id != INVALID_PLAYER
else None,
"ball_owning_team": None,
"ball_state": None,
"period": periods[-1],
"timestamp": raw_event["eventSec"],
}
event = None
if raw_event["eventName"] == wyscout_events.SHOT.EVENT:
shot_event_args = _parse_shot(raw_event, next_event)
event = ShotEvent.create(
**shot_event_args, **generic_event_args
)
elif raw_event["eventName"] == wyscout_events.PASS.EVENT:
pass_event_args = _parse_pass(raw_event, next_event)
event = PassEvent.create(
**pass_event_args, **generic_event_args
)
elif raw_event["eventName"] == wyscout_events.FOUL.EVENT:
foul_event_args = _parse_foul(raw_event)
event = FoulCommittedEvent.create(
**foul_event_args, **generic_event_args
)
if any(
(_has_tag(raw_event, tag) for tag in wyscout_tags.CARD)
):
card_event_args = _parse_card(raw_event)
event = CardEvent.create(
**card_event_args, **generic_event_args
)
elif (
raw_event["eventName"] == wyscout_events.INTERRUPTION.EVENT
):
ball_out_event_args = _parse_ball_out(raw_event)
event = BallOutEvent.create(
**ball_out_event_args, **generic_event_args
)
elif raw_event["eventName"] == wyscout_events.FREE_KICK.EVENT:
set_piece_event_args = _parse_set_piece(
raw_event, next_event
)
if (
raw_event["subEventName"]
in wyscout_events.FREE_KICK.PASS_TYPES
):
event = PassEvent.create(
**set_piece_event_args, **generic_event_args
)
elif (
raw_event["subEventName"]
in wyscout_events.FREE_KICK.SHOT_TYPES
):
event = ShotEvent.create(
**set_piece_event_args, **generic_event_args
)
elif (
raw_event["eventName"]
== wyscout_events.OTHERS_ON_BALL.EVENT
):
recovery_event_args = _parse_recovery(raw_event)
event = RecoveryEvent.create(
**recovery_event_args, **generic_event_args
)
elif raw_event["eventName"] == wyscout_events.DUEL.EVENT:
takeon_event_args = _parse_takeon(raw_event)
event = TakeOnEvent.create(
**takeon_event_args, **generic_event_args
)
elif raw_event["eventName"] not in [
wyscout_events.SAVE.EVENT,
wyscout_events.OFFSIDE.EVENT,
]:
# The events SAVE and OFFSIDE are already merged with PASS and SHOT events
qualifiers = _generic_qualifiers(raw_event)
event = GenericEvent.create(
result=None,
qualifiers=qualifiers,
**generic_event_args
)
if event and _include_event(event, wanted_event_types):
events.append(event)
metadata = Metadata(
teams=[home_team, away_team],
periods=periods,
pitch_dimensions=PitchDimensions(
x_dim=Dimension(0, 100), y_dim=Dimension(0, 100)
),
score=None,
frame_rate=None,
orientation=Orientation.BALL_OWNING_TEAM,
flags=None,
provider=Provider.WYSCOUT,
)
return EventDataset(metadata=metadata, records=events)
def deserialize(
self, inputs: Dict[str, Readable], options: Dict = None
) -> TrackingDataset:
"""
Deserialize TRACAB tracking data into a `TrackingDataset`.
Parameters
----------
inputs : dict
input `raw_data` should point to a `Readable` object containing
the 'csv' formatted raw data. input `metadata` should point to
the xml metadata data.
options : dict
Options for deserialization of the TRACAB file. Possible options are
`only_alive` (boolean) to specify that only frames with alive ball state
should be loaded, or `sample_rate` (float between 0 and 1) to specify
the amount of frames that should be loaded, `limit` to specify the max number of
frames that will be returned.
Returns
-------
dataset : TrackingDataset
Raises
------
-
See Also
--------
Examples
--------
>>> serializer = TRACABSerializer()
>>> with open("metadata.xml", "rb") as meta, \
>>> open("raw.dat", "rb") as raw:
>>> dataset = serializer.deserialize(
>>> inputs={
>>> 'metadata': meta,
>>> 'raw_data': raw
>>> },
>>> options={
>>> 'only_alive': True,
>>> 'sample_rate': 1/12
>>> }
>>> )
"""
self.__validate_inputs(inputs)
if not options:
options = {}
sample_rate = float(options.get("sample_rate", 1.0))
limit = int(options.get("limit", 0))
only_alive = bool(options.get("only_alive", True))
# TODO: also used in Metrica, extract to a method
home_team = Team(team_id="home", name="home", ground=Ground.HOME)
away_team = Team(team_id="away", name="away", ground=Ground.AWAY)
teams = [home_team, away_team]
with performance_logging("Loading metadata", logger=logger):
match = objectify.fromstring(inputs["metadata"].read()).match
frame_rate = int(match.attrib["iFrameRateFps"])
pitch_size_width = float(match.attrib["fPitchXSizeMeters"])
pitch_size_height = float(match.attrib["fPitchYSizeMeters"])
periods = []
for period in match.iterchildren(tag="period"):
start_frame_id = int(period.attrib["iStartFrame"])
end_frame_id = int(period.attrib["iEndFrame"])
if start_frame_id != 0 or end_frame_id != 0:
periods.append(
Period(
id=int(period.attrib["iId"]),
start_timestamp=start_frame_id / frame_rate,
end_timestamp=end_frame_id / frame_rate,
)
)
with performance_logging("Loading data", logger=logger):
def _iter():
n = 0
sample = 1.0 / sample_rate
for line_ in inputs["raw_data"].readlines():
line_ = line_.strip().decode("ascii")
if not line_:
continue
frame_id = int(line_[:10].split(":", 1)[0])
if only_alive and not line_.endswith("Alive;:"):
continue
for period_ in periods:
if period_.contains(frame_id / frame_rate):
if n % sample == 0:
yield period_, line_
n += 1
frames = []
for n, (period, line) in enumerate(_iter()):
frame = self._frame_from_line(teams, period, line, frame_rate)
frames.append(frame)
if not period.attacking_direction_set:
period.set_attacking_direction(
attacking_direction=attacking_direction_from_frame(
frame
)
)
if limit and n >= limit:
break
orientation = (
Orientation.FIXED_HOME_AWAY
if periods[0].attacking_direction == AttackingDirection.HOME_AWAY
else Orientation.FIXED_AWAY_HOME
)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=PitchDimensions(
x_dim=Dimension(
-1 * pitch_size_width / 2, pitch_size_width / 2
),
y_dim=Dimension(
-1 * pitch_size_height / 2, pitch_size_height / 2
),
x_per_meter=100,
y_per_meter=100,
),
score=None,
frame_rate=frame_rate,
orientation=orientation,
provider=Provider.TRACAB,
flags=DatasetFlag.BALL_OWNING_TEAM | DatasetFlag.BALL_STATE,
)
return TrackingDataset(
records=frames,
metadata=metadata,
)
def deserialize(
self, inputs: Dict[str, Readable], options: Dict = None
) -> EventDataset:
"""
Deserialize Metrica Sports event data json format into a `EventDataset`.
Parameters
----------
inputs : dict
input `event_data` should point to a `Readable` object containing
the 'json' formatted event data. input `metadata` should point
to a `Readable` object containing the `xml` metadata file.
options : dict
Options for deserialization of the Metrica Sports event json file.
Possible options are `event_types` (list of event types) to specify
the event types that should be returned. Valid types: "shot", "pass",
"carry", "take_on" and "generic". Generic is everything other than
the first 4. Those events are barely parsed. This type of event can
be used to do the parsing yourself.
Every event has a 'raw_event' attribute which contains the original
dictionary.
Returns
-------
dataset : EventDataset
Raises
------
See Also
--------
Examples
--------
>>> serializer = MetricaEventsJsonSerializer()
>>> with open("events.json", "rb") as event_data, \
>>> open("metadata.xml", "rb") as metadata:
>>>
>>> dataset = serializer.deserialize(
>>> inputs={
>>> 'event_data': event_data,
>>> 'metadata': metadata
>>> },
>>> options={
>>> 'event_types': ["pass", "take_on", "carry", "shot"]
>>> }
>>> )
"""
self.__validate_inputs(inputs)
if not options:
options = {}
with performance_logging("load data", logger=logger):
raw_events = json.load(inputs["event_data"])
metadata = load_metadata(
inputs["metadata"], provider=Provider.METRICA
)
with performance_logging("parse data", logger=logger):
wanted_event_types = [
EventType[event_type.upper()]
for event_type in options.get("event_types", [])
]
events = []
for i, raw_event in enumerate(raw_events["data"]):
if raw_event["team"]["id"] == metadata.teams[0].team_id:
team = metadata.teams[0]
elif raw_event["team"]["id"] == metadata.teams[1].team_id:
team = metadata.teams[1]
else:
raise Exception(
f"Unknown team_id {raw_event['team']['id']}"
)
player = team.get_player_by_id(raw_event["from"]["id"])
event_type = raw_event["type"]["id"]
subtypes = _parse_subtypes(raw_event)
period = [
period
for period in metadata.periods
if period.id == raw_event["period"]
][0]
previous_event = raw_events["data"][i - 1]
generic_event_kwargs = dict(
# from DataRecord
period=period,
timestamp=raw_event["start"]["time"],
ball_owning_team=_parse_ball_owning_team(event_type, team),
ball_state=BallState.ALIVE,
# from Event
event_id=None,
team=team,
player=player,
coordinates=(_parse_coordinates(raw_event["start"])),
raw_event=raw_event,
)
iteration_events = []
if event_type in MS_PASS_TYPES:
pass_event_kwargs = _parse_pass(
event=raw_event,
previous_event=previous_event,
subtypes=subtypes,
team=team,
)
event = PassEvent.create(
**pass_event_kwargs,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_SHOT:
shot_event_kwargs = _parse_shot(
event=raw_event,
previous_event=previous_event,
subtypes=subtypes,
)
event = ShotEvent.create(
**shot_event_kwargs,
**generic_event_kwargs,
)
elif subtypes and MS_EVENT_TYPE_DRIBBLE in subtypes:
take_on_event_kwargs = _parse_take_on(subtypes=subtypes)
event = TakeOnEvent.create(
qualifiers=None,
**take_on_event_kwargs,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_CARRY:
carry_event_kwargs = _parse_carry(
event=raw_event,
)
event = CarryEvent.create(
qualifiers=None,
**carry_event_kwargs,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_RECOVERY:
event = RecoveryEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_FOUL_COMMITTED:
event = FoulCommittedEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
else:
event = GenericEvent.create(
result=None,
qualifiers=None,
event_name=raw_event["type"]["name"],
**generic_event_kwargs,
)
if _include_event(event, wanted_event_types):
events.append(event)
# Checks if the event ended out of the field and adds a synthetic out event
if event.result in OUT_EVENT_RESULTS:
generic_event_kwargs["ball_state"] = BallState.DEAD
if raw_event["end"]["x"]:
generic_event_kwargs[
"coordinates"
] = _parse_coordinates(raw_event["end"])
generic_event_kwargs["timestamp"] = raw_event["end"][
"time"
]
event = BallOutEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
if _include_event(event, wanted_event_types):
events.append(event)
return EventDataset(
metadata=metadata,
records=events,
)
def deserialize(self, inputs: DatafactoryInputs) -> EventDataset:
transformer = self.get_transformer(length=2, width=2)
with performance_logging("load data", logger=logger):
data = json.load(inputs.event_data)
match = data["match"]
score_data = data["scoreStatus"]
incidences = data["incidences"]
players_data = data["players"]
teams_data = data["teams"]
with performance_logging("parse data", logger=logger):
teams = []
scores = []
team_ids = (
(Ground.HOME, str(match["homeTeamId"])),
(Ground.AWAY, str(match["awayTeamId"])),
)
for ground, team_id in team_ids:
team = Team(
team_id=team_id,
name=teams_data[team_id]["name"],
ground=ground,
)
team.players = [
Player(
player_id=player_id,
team=team,
first_name=player["name"]["first"],
last_name=player["name"]["last"],
name=player["name"]["shortName"]
or player["name"]["nick"],
jersey_no=player["squadNo"],
starting=not player["substitute"],
) for player_id, player in players_data.items()
if str(player["teamId"]) == team_id
]
teams.append(team)
scores.append(score_data.get(team_id, {}).get("score"))
score = Score(home=scores[0], away=scores[1])
# setup periods
status = incidences.pop(DF_EVENT_CLASS_STATUS)
# start timestamps are fixed
start_ts = {1: 0, 2: 45 * 60, 3: 90 * 60, 4: 105 * 60, 5: 120 * 60}
# check for end status updates to setup periods
end_event_types = {
DF_EVENT_TYPE_STATUS_MATCH_END,
DF_EVENT_TYPE_STATUS_FIRST_HALF_END,
DF_EVENT_TYPE_STATUS_SECOND_HALF_END,
DF_EVENT_TYPE_STATUS_FIRST_EXTRA_END,
DF_EVENT_TYPE_STATUS_SECOND_EXTRA_END,
}
periods = {}
for status_update in status.values():
if status_update["type"] not in end_event_types:
continue
half = status_update["t"]["half"]
end_ts = parse_str_ts(status_update)
periods[half] = Period(
id=half,
start_timestamp=start_ts[half],
end_timestamp=end_ts,
attacking_direction=AttackingDirection.HOME_AWAY if half %
2 == 1 else AttackingDirection.AWAY_HOME,
)
# exclude goals, already listed as shots too
incidences.pop(DF_EVENT_CLASS_GOALS)
raw_events = [(k, e_id, e) for k in incidences
for e_id, e in incidences[k].items()]
# sort events by timestamp, event_id
raw_events.sort(key=lambda e: (
e[2]["t"]["half"],
e[2]["t"]["m"],
e[2]["t"]["s"] or 0,
e[1],
))
home_team, away_team = teams
events = []
previous_event = next_event = None
for i, (e_class, e_id, raw_event) in enumerate(raw_events):
period = periods.get(raw_event["t"]["half"])
if period is None:
# skip invalid event
continue
timestamp = parse_str_ts(raw_event)
if (previous_event is not None and
previous_event["t"]["half"] != raw_event["t"]["half"]):
previous_event = None
next_event = (raw_events[i + 1][2]
if i + 1 < len(raw_events) else None)
team, player = _get_team_and_player(raw_event, home_team,
away_team)
event_base_kwargs = dict(
# from DataRecord
period=period,
timestamp=timestamp,
ball_owning_team=team,
ball_state=BallState.ALIVE,
# from Event
event_id=e_id,
team=team,
player=player,
coordinates=(_parse_coordinates(raw_event["coord"]["1"])
if "coord" in raw_event else None),
raw_event=raw_event,
result=None,
qualifiers=None,
)
if e_class in DF_EVENT_CLASS_PASSES:
pass_event_kwargs = _parse_pass(
raw_event=raw_event,
team=team,
previous_event=previous_event,
next_event=next_event,
)
event_base_kwargs.update(pass_event_kwargs)
event = PassEvent.create(**event_base_kwargs)
elif e_class == DF_EVENT_CLASS_SHOTS:
shot_event_kwargs = _parse_shot(
raw_event=raw_event,
previous_event=previous_event,
)
event_base_kwargs.update(shot_event_kwargs)
event = ShotEvent.create(**event_base_kwargs)
elif e_class == DF_EVENT_CLASS_STEALINGS:
event = RecoveryEvent.create(**event_base_kwargs)
elif e_class == DF_EVENT_CLASS_FOULS:
# NOTE: could use qualifiers? (hand, foul, penalty?)
# switch possession team
event_base_kwargs["ball_owning_team"] = (
home_team if team == away_team else away_team)
event = FoulCommittedEvent.create(**event_base_kwargs)
elif e_class in DF_EVENT_CLASS_CARDS:
card_kwargs = _parse_card(raw_event=raw_event, )
event_base_kwargs.update(card_kwargs)
event = CardEvent.create(**event_base_kwargs)
elif e_class == DF_EVENT_CLASS_SUBSTITUTIONS:
substitution_event_kwargs = _parse_substitution(
raw_event=raw_event, team=team)
event_base_kwargs.update(substitution_event_kwargs)
event = SubstitutionEvent.create(**event_base_kwargs)
else:
# otherwise, a generic event
event = GenericEvent.create(
event_name=e_class,
**event_base_kwargs,
)
# check if the event implies ball was out of the field and add a synthetic out event
if raw_event["type"] in BALL_OUT_EVENTS:
ball_out_event = BallOutEvent.create(
# from DataRecord
period=period,
timestamp=timestamp,
ball_owning_team=team,
ball_state=BallState.DEAD,
# from Event
event_id=e_id,
team=team,
player=player,
coordinates=event.coordinates,
raw_event=raw_event,
result=None,
qualifiers=None,
)
if self.should_include_event(event):
events.append(
transformer.transform_event(ball_out_event))
if self.should_include_event(event):
events.append(transformer.transform_event(event))
# only consider as a previous_event a ball-in-play event
if e_class not in (
DF_EVENT_CLASS_YELLOW_CARDS,
DF_EVENT_CLASS_RED_CARDS,
DF_EVENT_CLASS_SUBSTITUTIONS,
DF_EVENT_CLASS_PENALTY_SHOOTOUT,
):
previous_event = raw_event
metadata = Metadata(
teams=teams,
periods=sorted(periods.values(), key=lambda p: p.id),
pitch_dimensions=transformer.get_to_coordinate_system().
pitch_dimensions,
frame_rate=None,
orientation=Orientation.HOME_TEAM,
flags=DatasetFlag.BALL_OWNING_TEAM,
score=score,
provider=Provider.DATAFACTORY,
coordinate_system=transformer.get_to_coordinate_system(),
)
return EventDataset(
metadata=metadata,
records=events,
)
def deserialize(self, inputs: SportecInputs) -> EventDataset:
with performance_logging("load data", logger=logger):
match_root = objectify.fromstring(inputs.meta_data.read())
event_root = objectify.fromstring(inputs.event_data.read())
with performance_logging("parse data", logger=logger):
x_max = float(
match_root.MatchInformation.Environment.attrib["PitchX"]
)
y_max = float(
match_root.MatchInformation.Environment.attrib["PitchY"]
)
transformer = self.get_transformer(length=x_max, width=y_max)
team_path = objectify.ObjectPath(
"PutDataRequest.MatchInformation.Teams"
)
team_elms = list(team_path.find(match_root).iterchildren("Team"))
for team_elm in team_elms:
if team_elm.attrib["Role"] == "home":
home_team = _team_from_xml_elm(team_elm)
elif team_elm.attrib["Role"] == "guest":
away_team = _team_from_xml_elm(team_elm)
else:
raise DeserializationError(
f"Unknown side: {team_elm.attrib['Role']}"
)
(
home_score,
away_score,
) = match_root.MatchInformation.General.attrib["Result"].split(":")
score = Score(home=int(home_score), away=int(away_score))
teams = [home_team, away_team]
if len(home_team.players) == 0 or len(away_team.players) == 0:
raise DeserializationError("LineUp incomplete")
periods = []
period_id = 0
events = []
for event_elm in event_root.iterchildren("Event"):
event_chain = _event_chain_from_xml_elm(event_elm)
timestamp = _parse_datetime(event_chain["Event"]["EventTime"])
if (
SPORTEC_EVENT_NAME_KICKOFF in event_chain
and "GameSection"
in event_chain[SPORTEC_EVENT_NAME_KICKOFF]
):
period_id += 1
period = Period(
id=period_id,
start_timestamp=timestamp,
end_timestamp=None,
)
if period_id == 1:
team_left = event_chain[SPORTEC_EVENT_NAME_KICKOFF][
"TeamLeft"
]
if team_left == home_team.team_id:
# goal of home team is on the left side.
# this means they attack from left to right
orientation = Orientation.FIXED_HOME_AWAY
period.set_attacking_direction(
AttackingDirection.HOME_AWAY
)
else:
orientation = Orientation.FIXED_AWAY_HOME
period.set_attacking_direction(
AttackingDirection.AWAY_HOME
)
else:
last_period = periods[-1]
period.set_attacking_direction(
AttackingDirection.AWAY_HOME
if last_period.attacking_direction
== AttackingDirection.HOME_AWAY
else AttackingDirection.HOME_AWAY
)
periods.append(period)
elif SPORTEC_EVENT_NAME_FINAL_WHISTLE in event_chain:
period.end_timestamp = timestamp
continue
team = None
player = None
flatten_attributes = dict()
# reverse because top levels are more important
for event_attributes in reversed(event_chain.values()):
flatten_attributes.update(event_attributes)
if "Team" in flatten_attributes:
team = (
home_team
if flatten_attributes["Team"] == home_team.team_id
else away_team
)
if "Player" in flatten_attributes:
if not team:
raise ValueError("Player set while team is not set")
player = team.get_player_by_id(
flatten_attributes["Player"]
)
generic_event_kwargs = dict(
# from DataRecord
period=period,
timestamp=timestamp - period.start_timestamp,
ball_owning_team=None,
ball_state=BallState.ALIVE,
# from Event
event_id=event_chain["Event"]["EventId"],
coordinates=_parse_coordinates(event_chain["Event"]),
raw_event=flatten_attributes,
team=team,
player=player,
)
event_name, event_attributes = event_chain.popitem()
if event_name in SPORTEC_SHOT_EVENT_NAMES:
shot_event_kwargs = _parse_shot(
event_name=event_name, event_chain=event_chain
)
event = ShotEvent.create(
**shot_event_kwargs,
**generic_event_kwargs,
)
elif event_name in SPORTEC_PASS_EVENT_NAMES:
pass_event_kwargs = _parse_pass(
event_chain=event_chain, team=team
)
event = PassEvent.create(
**pass_event_kwargs,
**generic_event_kwargs,
receive_timestamp=None,
receiver_coordinates=None,
)
elif event_name == SPORTEC_EVENT_NAME_BALL_CLAIMING:
event = RecoveryEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif event_name == SPORTEC_EVENT_NAME_SUBSTITUTION:
substitution_event_kwargs = _parse_substitution(
event_attributes=event_attributes, team=team
)
generic_event_kwargs["player"] = substitution_event_kwargs[
"player"
]
del substitution_event_kwargs["player"]
event = SubstitutionEvent.create(
result=None,
qualifiers=None,
**substitution_event_kwargs,
**generic_event_kwargs,
)
elif event_name == SPORTEC_EVENT_NAME_CAUTION:
card_kwargs = _parse_caution(event_attributes)
event = CardEvent.create(
result=None,
qualifiers=None,
**card_kwargs,
**generic_event_kwargs,
)
elif event_name == SPORTEC_EVENT_NAME_FOUL:
foul_kwargs = _parse_foul(event_attributes, teams=teams)
generic_event_kwargs.update(foul_kwargs)
event = FoulCommittedEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
else:
event = GenericEvent.create(
result=None,
qualifiers=None,
event_name=event_name,
**generic_event_kwargs,
)
if events:
previous_event = events[-1]
if (
previous_event.event_type == EventType.PASS
and previous_event.result == PassResult.COMPLETE
):
if "X-Source-Position" in event_chain["Event"]:
previous_event.receiver_coordinates = Point(
x=float(
event_chain["Event"]["X-Source-Position"]
),
y=float(
event_chain["Event"]["Y-Source-Position"]
),
)
if (
event.event_type == EventType.PASS
and event.get_qualifier_value(SetPieceQualifier)
in (
SetPieceType.THROW_IN,
SetPieceType.GOAL_KICK,
SetPieceType.CORNER_KICK,
)
):
# 1. update previous pass
if events[-1].event_type == EventType.PASS:
events[-1].result = PassResult.OUT
# 2. add synthetic out event
decision_timestamp = _parse_datetime(
event_chain[list(event_chain.keys())[1]][
"DecisionTimestamp"
]
)
out_event = BallOutEvent.create(
period=period,
timestamp=decision_timestamp - period.start_timestamp,
ball_owning_team=None,
ball_state=BallState.DEAD,
# from Event
event_id=event_chain["Event"]["EventId"] + "-ball-out",
team=events[-1].team,
player=events[-1].player,
coordinates=None,
raw_event={},
result=None,
qualifiers=None,
)
events.append(transformer.transform_event(out_event))
events.append(transformer.transform_event(event))
events = list(
filter(
self.should_include_event,
events,
)
)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=transformer.get_to_coordinate_system().pitch_dimensions,
score=score,
frame_rate=None,
orientation=orientation,
flags=~(DatasetFlag.BALL_STATE | DatasetFlag.BALL_OWNING_TEAM),
provider=Provider.SPORTEC,
coordinate_system=transformer.get_to_coordinate_system(),
)
return EventDataset(
metadata=metadata,
records=events,
)
def run_query(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(description="Run query on event data")
parser.add_argument(
"--input-statsbomb",
help="StatsBomb event input files (events.json,lineup.json)",
)
parser.add_argument("--input-opta",
help="Opta event input files (f24.xml,f7.xml)")
parser.add_argument("--input-datafactory",
help="Datafactory event input file (.json)")
parser.add_argument("--input-wyscout", help="Wyscout event input file")
parser.add_argument("--output-xml", help="Output file")
parser.add_argument(
"--with-success",
default=True,
help="Input existence of success capture in output",
)
parser.add_argument("--prepend-time",
default=7,
help="Seconds to prepend to match")
parser.add_argument("--append-time",
default=5,
help="Seconds to append to match")
parser.add_argument("--query-file",
help="File containing the query",
required=True)
parser.add_argument(
"--stats",
default="none",
help="Show matches stats",
choices=["text", "json", "none"],
)
parser.add_argument(
"--show-events",
default=False,
help="Show events for each match",
action="store_true",
)
parser.add_argument(
"--only-success",
default=False,
help="Only show/output success cases",
action="store_true",
)
logger = logging.getLogger("run_query")
logging.basicConfig(
stream=sys.stderr,
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
opts = parser.parse_args(argv)
query = load_query(opts.query_file)
dataset = None
if opts.input_statsbomb:
with performance_logging("load dataset", logger=logger):
events_filename, lineup_filename = opts.input_statsbomb.split(",")
dataset = statsbomb.load(
event_data=events_filename.strip(),
lineup_data=lineup_filename.strip(),
event_types=query.event_types,
)
if opts.input_opta:
with performance_logging("load dataset", logger=logger):
f24_filename, f7_filename = opts.input_opta.split(",")
dataset = opta.load(
f24_data=f24_filename.strip(),
f7_data=f7_filename.strip(),
event_types=query.event_types,
)
if opts.input_datafactory:
with performance_logging("load dataset", logger=logger):
events_filename = opts.input_datafactory
dataset = datafactory.load(
event_data=events_filename.strip(),
event_types=query.event_types,
)
if opts.input_wyscout:
with performance_logging("load dataset", logger=logger):
events_filename = opts.input_wyscout
dataset = wyscout.load(
event_data=events_filename,
event_types=query.event_types,
)
if not dataset:
raise Exception("You have to specify a dataset.")
with performance_logging("searching", logger=logger):
matches = pm.search(dataset, query.pattern)
# Construct new code dataset with same properties (eg periods)
# as original event dataset.
# Records will be added later below
records = []
counter = Counter()
for i, match in enumerate(matches):
team = match.events[0].team
success = "success" in match.captures
counter.update({
f"{team.ground}_total": 1,
f"{team.ground}_success": 1 if success else 0,
})
should_process = not opts.only_success or success
if opts.show_events and should_process:
print_match(i, match, success, str(team))
if opts.output_xml and should_process:
code_ = str(team)
if opts.with_success and success:
code_ += " success"
code = Code(
period=match.events[0].period,
code_id=str(i),
code=code_,
timestamp=match.events[0].timestamp - opts.prepend_time,
end_timestamp=match.events[-1].timestamp + opts.append_time,
# TODO: refactor those two out
ball_state=None,
ball_owning_team=None,
)
records.append(code)
code_dataset = CodeDataset(metadata=dataset.metadata, records=records)
if opts.output_xml:
sportscode.save(code_dataset, opts.output_xml)
logger.info(f"Wrote {len(code_dataset.codes)} video fragments to file")
if opts.stats == "text":
text_stats = """\
Home:
total count: {home_total}
success: {home_success} ({home_success_rate:.0f}%)
no success: {home_failure} ({home_failure_rate:.0f}%)
Away:
total count: {away_total}
success: {away_success} ({away_success_rate:.0f}%)
no success: {away_failure} ({away_failure_rate:.0f}%)
""".format(
home_total=counter["home_total"],
home_success=counter["home_success"],
home_success_rate=(counter["home_success"] /
counter["home_total"] * 100),
home_failure=counter["home_total"] - counter["home_success"],
home_failure_rate=(
(counter["home_total"] - counter["home_success"]) /
counter["home_total"] * 100),
away_total=counter["away_total"],
away_success=counter["away_success"],
away_success_rate=(counter["away_success"] /
counter["away_total"] * 100),
away_failure=counter["away_total"] - counter["away_success"],
away_failure_rate=(
(counter["away_total"] - counter["away_success"]) /
counter["away_total"] * 100),
)
print(textwrap.dedent(text_stats))
elif opts.stats == "json":
import json
print(json.dumps(counter, indent=4))
def deserialize(self, inputs: StatsbombInputs) -> EventDataset:
transformer = self.get_transformer(length=120, width=80)
with performance_logging("load data", logger=logger):
raw_events = json.load(inputs.event_data)
home_lineup, away_lineup = json.load(inputs.lineup_data)
(
shot_fidelity_version,
xy_fidelity_version,
) = _determine_xy_fidelity_versions(raw_events)
logger.info(
f"Determined Fidelity versions: shot v{shot_fidelity_version} / XY v{xy_fidelity_version}"
)
with performance_logging("parse data", logger=logger):
starting_player_ids = {
str(player["player"]["id"])
for raw_event in raw_events
if raw_event["type"]["id"] == SB_EVENT_TYPE_STARTING_XI
for player in raw_event["tactics"]["lineup"]
}
starting_formations = {
raw_event["team"]["id"]: FormationType("-".join(
list(str(raw_event["tactics"]["formation"]))))
for raw_event in raw_events
if raw_event["type"]["id"] == SB_EVENT_TYPE_STARTING_XI
}
home_team = Team(
team_id=str(home_lineup["team_id"]),
name=home_lineup["team_name"],
ground=Ground.HOME,
starting_formation=starting_formations[home_lineup["team_id"]],
)
home_team.players = [
Player(
player_id=str(player["player_id"]),
team=home_team,
name=player["player_name"],
jersey_no=int(player["jersey_number"]),
starting=str(player["player_id"]) in starting_player_ids,
) for player in home_lineup["lineup"]
]
away_team = Team(
team_id=str(away_lineup["team_id"]),
name=away_lineup["team_name"],
ground=Ground.AWAY,
starting_formation=starting_formations[away_lineup["team_id"]],
)
away_team.players = [
Player(
player_id=str(player["player_id"]),
team=away_team,
name=player["player_name"],
jersey_no=int(player["jersey_number"]),
starting=str(player["player_id"]) in starting_player_ids,
) for player in away_lineup["lineup"]
]
teams = [home_team, away_team]
periods = []
period = None
events = []
for raw_event in raw_events:
if raw_event["team"]["id"] == home_lineup["team_id"]:
team = home_team
elif raw_event["team"]["id"] == away_lineup["team_id"]:
team = away_team
else:
raise DeserializationError(
f"Unknown team_id {raw_event['team']['id']}")
if (raw_event["possession_team"]["id"] ==
home_lineup["team_id"]):
possession_team = home_team
elif (raw_event["possession_team"]["id"] ==
away_lineup["team_id"]):
possession_team = away_team
else:
raise DeserializationError(
f"Unknown possession_team_id: {raw_event['possession_team']}"
)
timestamp = parse_str_ts(raw_event["timestamp"])
period_id = int(raw_event["period"])
if not period or period.id != period_id:
period = Period(
id=period_id,
start_timestamp=(
timestamp if not period
# period = [start, end], add millisecond to prevent overlapping
else timestamp + period.end_timestamp + 0.001),
end_timestamp=None,
)
periods.append(period)
else:
period.end_timestamp = period.start_timestamp + timestamp
player = None
if "player" in raw_event:
player = team.get_player_by_id(raw_event["player"]["id"])
event_type = raw_event["type"]["id"]
if event_type == SB_EVENT_TYPE_SHOT:
fidelity_version = shot_fidelity_version
elif event_type in (
SB_EVENT_TYPE_CARRY,
SB_EVENT_TYPE_DRIBBLE,
SB_EVENT_TYPE_PASS,
):
fidelity_version = xy_fidelity_version
else:
# TODO: Uh ohhhh.. don't know which one to pick
fidelity_version = xy_fidelity_version
generic_event_kwargs = {
# from DataRecord
"period":
period,
"timestamp":
timestamp,
"ball_owning_team":
possession_team,
"ball_state":
BallState.ALIVE,
# from Event
"event_id":
raw_event["id"],
"team":
team,
"player":
player,
"coordinates": (_parse_coordinates(
raw_event.get("location"),
fidelity_version,
) if "location" in raw_event else None),
"related_event_ids":
raw_event.get("related_events", []),
"raw_event":
raw_event,
}
new_events = []
if event_type == SB_EVENT_TYPE_PASS:
pass_event_kwargs = _parse_pass(
pass_dict=raw_event["pass"],
team=team,
fidelity_version=fidelity_version,
)
pass_event = PassEvent.create(
# TODO: Consider moving this to _parse_pass
receive_timestamp=timestamp + raw_event["duration"],
**pass_event_kwargs,
**generic_event_kwargs,
)
new_events.append(pass_event)
elif event_type == SB_EVENT_TYPE_SHOT:
shot_event_kwargs = _parse_shot(
shot_dict=raw_event["shot"], )
shot_event = ShotEvent.create(
**shot_event_kwargs,
**generic_event_kwargs,
)
new_events.append(shot_event)
# For dribble and carry the definitions
# are flipped between Statsbomb and kloppy
elif event_type == SB_EVENT_TYPE_DRIBBLE:
take_on_event_kwargs = _parse_take_on(
take_on_dict=raw_event["dribble"], )
take_on_event = TakeOnEvent.create(
qualifiers=None,
**take_on_event_kwargs,
**generic_event_kwargs,
)
new_events.append(take_on_event)
elif event_type == SB_EVENT_TYPE_CARRY:
carry_event_kwargs = _parse_carry(
carry_dict=raw_event["carry"],
fidelity_version=fidelity_version,
)
carry_event = CarryEvent.create(
qualifiers=None,
# TODO: Consider moving this to _parse_carry
end_timestamp=timestamp + raw_event.get("duration", 0),
**carry_event_kwargs,
**generic_event_kwargs,
)
new_events.append(carry_event)
# lineup affecting events
elif event_type == SB_EVENT_TYPE_SUBSTITUTION:
substitution_event_kwargs = _parse_substitution(
substitution_dict=raw_event["substitution"],
team=team,
)
substitution_event = SubstitutionEvent.create(
result=None,
qualifiers=None,
**substitution_event_kwargs,
**generic_event_kwargs,
)
new_events.append(substitution_event)
elif event_type == SB_EVENT_TYPE_BAD_BEHAVIOUR:
bad_behaviour_kwargs = _parse_bad_behaviour(
bad_behaviour_dict=raw_event.get("bad_behaviour",
{}), )
if "card" in bad_behaviour_kwargs:
card_kwargs = bad_behaviour_kwargs["card"]
card_event = CardEvent.create(
result=None,
qualifiers=None,
card_type=card_kwargs["card_type"],
**generic_event_kwargs,
)
new_events.append(card_event)
elif event_type == SB_EVENT_TYPE_FOUL_COMMITTED:
foul_committed_kwargs = _parse_foul_committed(
foul_committed_dict=raw_event.get(
"foul_committed", {}), )
foul_committed_event = FoulCommittedEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
new_events.append(foul_committed_event)
if "card" in foul_committed_kwargs:
card_kwargs = foul_committed_kwargs["card"]
card_event = CardEvent.create(
result=None,
qualifiers=None,
card_type=card_kwargs["card_type"],
**generic_event_kwargs,
)
new_events.append(card_event)
elif event_type == SB_EVENT_TYPE_PLAYER_ON:
player_on_event = PlayerOnEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
new_events.append(player_on_event)
elif event_type == SB_EVENT_TYPE_PLAYER_OFF:
player_off_event = PlayerOffEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
new_events.append(player_off_event)
elif event_type == SB_EVENT_TYPE_RECOVERY:
recovery_event = RecoveryEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
new_events.append(recovery_event)
elif event_type == SB_EVENT_TYPE_FORMATION_CHANGE:
formation_change_event_kwargs = _parse_formation_change(
raw_event["tactics"]["formation"])
formation_change_event = FormationChangeEvent.create(
result=None,
qualifiers=None,
**formation_change_event_kwargs,
**generic_event_kwargs,
)
new_events.append(formation_change_event)
# rest: generic
else:
generic_event = GenericEvent.create(
result=None,
qualifiers=None,
event_name=raw_event["type"]["name"],
**generic_event_kwargs,
)
new_events.append(generic_event)
for event in new_events:
if self.should_include_event(event):
transformed_event = transformer.transform_event(event)
events.append(transformed_event)
# Checks if the event ended out of the field and adds a synthetic out event
if event.result in OUT_EVENT_RESULTS:
generic_event_kwargs["ball_state"] = BallState.DEAD
if event.receiver_coordinates:
generic_event_kwargs[
"coordinates"] = event.receiver_coordinates
ball_out_event = BallOutEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
if self.should_include_event(ball_out_event):
transformed_ball_out_event = (
transformer.transform_event(ball_out_event)
)
events.append(transformed_ball_out_event)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=transformer.get_to_coordinate_system().
pitch_dimensions,
frame_rate=None,
orientation=Orientation.ACTION_EXECUTING_TEAM,
flags=DatasetFlag.BALL_OWNING_TEAM,
score=None,
provider=Provider.STATSBOMB,
coordinate_system=transformer.get_to_coordinate_system(),
)
return EventDataset(
metadata=metadata,
records=events,
)
def deserialize(self,
inputs: Dict[str, Readable],
options: Dict = None) -> EventDataset:
"""
Deserialize Opta event data into a `EventDataset`.
Parameters
----------
inputs : dict
input `f24_data` should point to a `Readable` object containing
the 'xml' formatted event data. input `f7_data` should point
to a `Readable` object containing the 'xml' formatted f7 data.
options : dict
Options for deserialization of the Opta file. Possible options are
`event_types` (list of event types) to specify the event types that
should be returned. Valid types: "shot", "pass", "carry", "take_on" and
"generic". Generic is everything other than the first 4. Those events
are barely parsed. This type of event can be used to do the parsing
yourself.
Every event has a 'raw_event' attribute which contains the original
dictionary.
Returns
-------
dataset : EventDataset
Raises
------
See Also
--------
Examples
--------
>>> serializer = OptaSerializer()
>>> with open("123_f24.xml", "rb") as f24_data, \
>>> open("123_f7.xml", "rb") as f7_data:
>>>
>>> dataset = serializer.deserialize(
>>> inputs={
>>> 'f24_data': f24_data,
>>> 'f7_data': f7_data
>>> },
>>> options={
>>> 'event_types': ["pass", "take_on", "carry", "shot"]
>>> }
>>> )
"""
self.__validate_inputs(inputs)
if not options:
options = {}
with performance_logging("load data", logger=logger):
f7_root = objectify.fromstring(inputs["f7_data"].read())
f24_root = objectify.fromstring(inputs["f24_data"].read())
wanted_event_types = [
EventType[event_type.upper()]
for event_type in options.get("event_types", [])
]
with performance_logging("parse data", logger=logger):
matchdata_path = objectify.ObjectPath(
"SoccerFeed.SoccerDocument.MatchData")
team_elms = list(
matchdata_path.find(f7_root).iterchildren("TeamData"))
home_score = None
away_score = None
for team_elm in team_elms:
if team_elm.attrib["Side"] == "Home":
home_score = team_elm.attrib["Score"]
home_team = _team_from_xml_elm(team_elm, f7_root)
elif team_elm.attrib["Side"] == "Away":
away_score = team_elm.attrib["Score"]
away_team = _team_from_xml_elm(team_elm, f7_root)
else:
raise Exception(f"Unknown side: {team_elm.attrib['Side']}")
score = Score(home=home_score, away=away_score)
teams = [home_team, away_team]
if len(home_team.players) == 0 or len(away_team.players) == 0:
raise Exception("LineUp incomplete")
game_elm = f24_root.find("Game")
periods = [
Period(
id=1,
start_timestamp=None,
end_timestamp=None,
),
Period(
id=2,
start_timestamp=None,
end_timestamp=None,
),
]
possession_team = None
events = []
for event_elm in game_elm.iterchildren("Event"):
event_id = event_elm.attrib["id"]
type_id = int(event_elm.attrib["type_id"])
timestamp = _parse_f24_datetime(event_elm.attrib["timestamp"])
period_id = int(event_elm.attrib["period_id"])
for period in periods:
if period.id == period_id:
break
else:
logger.debug(
f"Skipping event {event_id} because period doesn't match {period_id}"
)
continue
if type_id == EVENT_TYPE_START_PERIOD:
logger.debug(
f"Set start of period {period.id} to {timestamp}")
period.start_timestamp = timestamp
elif type_id == EVENT_TYPE_END_PERIOD:
logger.debug(
f"Set end of period {period.id} to {timestamp}")
period.end_timestamp = timestamp
else:
if not period.start_timestamp:
# not started yet
continue
if event_elm.attrib["team_id"] == home_team.team_id:
team = teams[0]
elif event_elm.attrib["team_id"] == away_team.team_id:
team = teams[1]
else:
raise Exception(
f"Unknown team_id {event_elm.attrib['team_id']}")
x = float(event_elm.attrib["x"])
y = float(event_elm.attrib["y"])
outcome = int(event_elm.attrib["outcome"])
raw_qualifiers = {
int(qualifier_elm.attrib["qualifier_id"]):
qualifier_elm.attrib.get("value")
for qualifier_elm in event_elm.iterchildren("Q")
}
player = None
if "player_id" in event_elm.attrib:
player = team.get_player_by_id(
event_elm.attrib["player_id"])
if type_id in BALL_OWNING_EVENTS:
possession_team = team
generic_event_kwargs = dict(
# from DataRecord
period=period,
timestamp=timestamp - period.start_timestamp,
ball_owning_team=possession_team,
ball_state=BallState.ALIVE,
# from Event
event_id=event_id,
team=team,
player=player,
coordinates=Point(x=x, y=y),
raw_event=event_elm,
)
if type_id == EVENT_TYPE_PASS:
pass_event_kwargs = _parse_pass(
raw_qualifiers, outcome)
event = PassEvent.create(
**pass_event_kwargs,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_OFFSIDE_PASS:
pass_event_kwargs = _parse_offside_pass(raw_qualifiers)
event = PassEvent.create(
**pass_event_kwargs,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_TAKE_ON:
take_on_event_kwargs = _parse_take_on(outcome)
event = TakeOnEvent.create(
qualifiers=None,
**take_on_event_kwargs,
**generic_event_kwargs,
)
elif type_id in (
EVENT_TYPE_SHOT_MISS,
EVENT_TYPE_SHOT_POST,
EVENT_TYPE_SHOT_SAVED,
EVENT_TYPE_SHOT_GOAL,
):
shot_event_kwargs = _parse_shot(
raw_qualifiers,
type_id,
coordinates=generic_event_kwargs["coordinates"],
)
kwargs = {}
kwargs.update(generic_event_kwargs)
kwargs.update(shot_event_kwargs)
event = ShotEvent.create(**kwargs)
elif type_id == EVENT_TYPE_RECOVERY:
event = RecoveryEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif type_id == EVENT_TYPE_FOUL_COMMITTED:
event = FoulCommittedEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif type_id in BALL_OUT_EVENTS:
generic_event_kwargs["ball_state"] = BallState.DEAD
event = BallOutEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
else:
event = GenericEvent.create(
**generic_event_kwargs,
result=None,
qualifiers=None,
event_name=_get_event_type_name(type_id),
)
if (not wanted_event_types
or event.event_type in wanted_event_types):
events.append(event)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=PitchDimensions(x_dim=Dimension(0, 100),
y_dim=Dimension(0, 100)),
score=score,
frame_rate=None,
orientation=Orientation.ACTION_EXECUTING_TEAM,
flags=DatasetFlag.BALL_OWNING_TEAM,
provider=Provider.OPTA,
)
return EventDataset(
metadata=metadata,
records=events,
)
def deserialize(self, inputs: MetricaJsonEventDataInputs) -> EventDataset:
with performance_logging("load data", logger=logger):
raw_events = json.load(inputs.event_data)
metadata = load_metadata(inputs.meta_data,
provider=Provider.METRICA)
transformer = self.get_transformer(
length=metadata.pitch_dimensions.length,
width=metadata.pitch_dimensions.width,
)
with performance_logging("parse data", logger=logger):
events = []
for i, raw_event in enumerate(raw_events["data"]):
if raw_event["team"]["id"] == metadata.teams[0].team_id:
team = metadata.teams[0]
elif raw_event["team"]["id"] == metadata.teams[1].team_id:
team = metadata.teams[1]
else:
raise DeserializationError(
f"Unknown team_id {raw_event['team']['id']}")
player = team.get_player_by_id(raw_event["from"]["id"])
event_type = raw_event["type"]["id"]
subtypes = _parse_subtypes(raw_event)
period = [
period for period in metadata.periods
if period.id == raw_event["period"]
][0]
previous_event = raw_events["data"][i - 1]
generic_event_kwargs = dict(
# from DataRecord
period=period,
timestamp=raw_event["start"]["time"],
ball_owning_team=_parse_ball_owning_team(event_type, team),
ball_state=BallState.ALIVE,
# from Event
event_id=None,
team=team,
player=player,
coordinates=(_parse_coordinates(raw_event["start"])),
raw_event=raw_event,
)
iteration_events = []
if event_type in MS_PASS_TYPES:
pass_event_kwargs = _parse_pass(
event=raw_event,
previous_event=previous_event,
subtypes=subtypes,
team=team,
)
event = PassEvent.create(
**pass_event_kwargs,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_SHOT:
shot_event_kwargs = _parse_shot(
event=raw_event,
previous_event=previous_event,
subtypes=subtypes,
)
event = ShotEvent.create(
**shot_event_kwargs,
**generic_event_kwargs,
)
elif subtypes and MS_EVENT_TYPE_DRIBBLE in subtypes:
take_on_event_kwargs = _parse_take_on(subtypes=subtypes)
event = TakeOnEvent.create(
qualifiers=None,
**take_on_event_kwargs,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_CARRY:
carry_event_kwargs = _parse_carry(event=raw_event, )
event = CarryEvent.create(
qualifiers=None,
**carry_event_kwargs,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_RECOVERY:
event = RecoveryEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
elif event_type == MS_EVENT_TYPE_FOUL_COMMITTED:
event = FoulCommittedEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
else:
event = GenericEvent.create(
result=None,
qualifiers=None,
event_name=raw_event["type"]["name"],
**generic_event_kwargs,
)
if self.should_include_event(event):
events.append(transformer.transform_event(event))
# Checks if the event ended out of the field and adds a synthetic out event
if event.result in OUT_EVENT_RESULTS:
generic_event_kwargs["ball_state"] = BallState.DEAD
if raw_event["end"]["x"]:
generic_event_kwargs[
"coordinates"] = _parse_coordinates(
raw_event["end"])
generic_event_kwargs["timestamp"] = raw_event["end"][
"time"]
event = BallOutEvent.create(
result=None,
qualifiers=None,
**generic_event_kwargs,
)
if self.should_include_event(event):
events.append(transformer.transform_event(event))
return EventDataset(
metadata=metadata,
records=events,
)
def deserialize(self, inputs: TRACABInputs) -> TrackingDataset:
# TODO: also used in Metrica, extract to a method
home_team = Team(team_id="home", name="home", ground=Ground.HOME)
away_team = Team(team_id="away", name="away", ground=Ground.AWAY)
teams = [home_team, away_team]
with performance_logging("Loading metadata", logger=logger):
match = objectify.fromstring(inputs.meta_data.read()).match
frame_rate = int(match.attrib["iFrameRateFps"])
pitch_size_width = float(match.attrib["fPitchXSizeMeters"])
pitch_size_height = float(match.attrib["fPitchYSizeMeters"])
periods = []
for period in match.iterchildren(tag="period"):
start_frame_id = int(period.attrib["iStartFrame"])
end_frame_id = int(period.attrib["iEndFrame"])
if start_frame_id != 0 or end_frame_id != 0:
periods.append(
Period(
id=int(period.attrib["iId"]),
start_timestamp=start_frame_id / frame_rate,
end_timestamp=end_frame_id / frame_rate,
)
)
with performance_logging("Loading data", logger=logger):
transformer = self.get_transformer(
length=pitch_size_width, width=pitch_size_height
)
def _iter():
n = 0
sample = 1.0 / self.sample_rate
for line_ in inputs.raw_data.readlines():
line_ = line_.strip().decode("ascii")
if not line_:
continue
frame_id = int(line_[:10].split(":", 1)[0])
if self.only_alive and not line_.endswith("Alive;:"):
continue
for period_ in periods:
if period_.contains(frame_id / frame_rate):
if n % sample == 0:
yield period_, line_
n += 1
frames = []
for n, (period, line) in enumerate(_iter()):
frame = self._frame_from_line(teams, period, line, frame_rate)
frame = transformer.transform_frame(frame)
frames.append(frame)
if not period.attacking_direction_set:
period.set_attacking_direction(
attacking_direction=attacking_direction_from_frame(
frame
)
)
if self.limit and n >= self.limit:
break
orientation = (
Orientation.FIXED_HOME_AWAY
if periods[0].attacking_direction == AttackingDirection.HOME_AWAY
else Orientation.FIXED_AWAY_HOME
)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=transformer.get_to_coordinate_system().pitch_dimensions,
score=None,
frame_rate=frame_rate,
orientation=orientation,
provider=Provider.TRACAB,
flags=DatasetFlag.BALL_OWNING_TEAM | DatasetFlag.BALL_STATE,
coordinate_system=transformer.get_to_coordinate_system(),
)
return TrackingDataset(
records=frames,
metadata=metadata,
)
def deserialize(self, inputs: SkillCornerInputs) -> TrackingDataset:
metadata = self.__load_json(inputs.meta_data)
raw_data = self.__load_json(inputs.raw_data)
with performance_logging("Loading metadata", logger=logger):
periods = self.__get_periods(raw_data)
teamdict = {
metadata["home_team"].get("id"): "home_team",
metadata["away_team"].get("id"): "away_team",
}
player_id_to_team_dict = {
player["trackable_object"]: player["team_id"]
for player in metadata["players"]
}
player_dict = {
player["trackable_object"]: player
for player in metadata["players"]
}
referee_dict = {
ref["trackable_object"]: "referee"
for ref in metadata["referees"]
}
ball_id = metadata["ball"]["trackable_object"]
# there are different pitch_sizes in SkillCorner
pitch_size_width = metadata["pitch_width"]
pitch_size_length = metadata["pitch_length"]
transformer = self.get_transformer(length=pitch_size_length,
width=pitch_size_width)
home_team_id = metadata["home_team"]["id"]
away_team_id = metadata["away_team"]["id"]
players = {"HOME": {}, "AWAY": {}}
home_team = Team(
team_id=home_team_id,
name=metadata["home_team"]["name"],
ground=Ground.HOME,
)
away_team = Team(
team_id=away_team_id,
name=metadata["away_team"]["name"],
ground=Ground.AWAY,
)
teams = [home_team, away_team]
for player_id in player_dict.keys():
player = player_dict.get(player_id)
team_id = player["team_id"]
if team_id == home_team_id:
team_string = "HOME"
team = home_team
elif team_id == away_team_id:
team_string = "AWAY"
team = away_team
players[team_string][player_id] = Player(
player_id=f"{team.ground}_{player['number']}",
team=team,
jersey_no=player["number"],
name=f"{player['first_name']} {player['last_name']}",
first_name=player["first_name"],
last_name=player["last_name"],
starting=player["start_time"] == "00:00:00",
position=Position(
position_id=player["player_role"].get("id"),
name=player["player_role"].get("name"),
coordinates=None,
),
attributes={},
)
home_team.players = list(players["HOME"].values())
away_team.players = list(players["AWAY"].values())
anon_players = {"HOME": {}, "AWAY": {}}
with performance_logging("Loading data", logger=logger):
def _iter():
n = 0
sample = 1.0 / self.sample_rate
for frame in raw_data:
frame_period = frame["period"]
if frame_period is not None:
if n % sample == 0:
yield frame
n += 1
frames = []
n_frames = 0
for _frame in _iter():
# include frame if there is any tracking data, players or ball.
# or if include_empty_frames == True
if self.include_empty_frames or len(_frame["data"]) > 0:
frame = self._get_frame_data(
teams,
teamdict,
players,
player_id_to_team_dict,
periods,
player_dict,
anon_players,
ball_id,
referee_dict,
_frame,
)
frame = transformer.transform_frame(frame)
frames.append(frame)
n_frames += 1
if self.limit and n_frames >= self.limit:
break
self._set_skillcorner_attacking_directions(frames, periods)
frame_rate = 10
orientation = (Orientation.HOME_TEAM if periods[1].attacking_direction
== AttackingDirection.HOME_AWAY else
Orientation.AWAY_TEAM)
metadata = Metadata(
teams=teams,
periods=periods,
pitch_dimensions=transformer.get_to_coordinate_system().
pitch_dimensions,
score=Score(
home=metadata["home_team_score"],
away=metadata["away_team_score"],
),
frame_rate=frame_rate,
orientation=orientation,
provider=Provider.SKILLCORNER,
flags=~(DatasetFlag.BALL_STATE | DatasetFlag.BALL_OWNING_TEAM),
coordinate_system=transformer.get_to_coordinate_system(),
)
return TrackingDataset(
records=frames,
metadata=metadata,
)
