def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
"""
Given a 2D DataFrame, ndarray or sparse equivalent, create and return a Matrix flatbuffer.
:param matrix: 2D DataFrame, ndarray or sparse equivalent
:param row_idx: index for row dimension, Index or ndarray
:param col_idx: index for col dimension, Index or ndarray
NOTE: row indices are (currently) unsupported and must be None
"""
if row_idx is not None:
raise ValueError("row indexing not supported for FBS Matrix")
if matrix.ndim != 2:
raise ValueError("FBS Matrix must be 2D")
(n_rows, n_cols) = matrix.shape
# estimate size needed, so we don't unnecessarily realloc.
builder = Builder(guess_at_mem_needed(matrix))
columns = []
for cidx in range(n_cols - 1, -1, -1):
# serialize the typed array
col = matrix.iloc[:, cidx] if isinstance(
matrix, pd.DataFrame) else matrix[:, cidx]
typed_arr = serialize_typed_array(builder, col, column_encoding)
# serialize the Column union
columns.append(serialize_column(builder, typed_arr))
# Serialize Matrix.columns[]
Matrix.MatrixStartColumnsVector(builder, n_cols)
for c in columns:
builder.PrependUOffsetTRelative(c)
matrix_column_vec = builder.EndVector(n_cols)
# serialize the colIndex if provided
cidx = None
if col_idx is not None:
cidx = serialize_typed_array(builder, col_idx, index_encoding)
# Serialize Matrix
matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx)
builder.Finish(matrix)
return builder.Output()
def build_feed_request(self, builder: flatbuffers.Builder) -> bytearray:
build_market = []
size = len(self.markets)
for market in self.markets:
build_market.append(builder.CreateString(market))
req_id_string = builder.CreateString(self.req_id)
FbsFr.FeedRequestStartMarketsVector(builder, size)
for b in reversed(build_market):
builder.PrependUOffsetTRelative(b)
markets = builder.EndVector(size)
FbsFr.FeedRequestStart(builder)
FbsFr.FeedRequestAddReqId(builder, req_id_string)
FbsFr.FeedRequestAddSubReqType(builder, self.sub_req_type)
FbsFr.FeedRequestAddFeedType(builder, self.feed_type)
FbsFr.FeedRequestAddGrouping(builder, self.grouping)
FbsFr.FeedRequestAddDepth(builder, self.depth)
FbsFr.FeedRequestAddFrequency(builder, self.frequency)
FbsFr.FeedRequestAddMarkets(builder, markets)
feed_req = FbsFr.FeedRequestEnd(builder)
FbsMessage.MessageStart(builder)
FbsMessage.MessageAddTts(builder, time_ns())
FbsMessage.MessageAddBodyType(builder, FbsBody.Body.FeedRequest)
FbsMessage.MessageAddBody(builder, feed_req)
msg = FbsMessage.MessageEnd(builder)
builder.Finish(msg)
return builder.Output()
def to_fb(self, builder: flatbuffers.Builder):
robot_audio_arr = []
for keyframe in self.keyframes:
if isinstance(keyframe, AnimRobotAudio):
fbkf = keyframe.to_fb(builder)
robot_audio_arr.append(fbkf)
CozmoAnim.Keyframes.KeyframesStartRobotAudioKeyFrameVector(
builder, len(robot_audio_arr))
for i in reversed(range(len(robot_audio_arr))):
builder.PrependUOffsetTRelative(robot_audio_arr[i])
robot_audio_vector = builder.EndVector(len(robot_audio_arr))
CozmoAnim.Keyframes.KeyframesStart(builder)
CozmoAnim.Keyframes.KeyframesAddRobotAudioKeyFrame(
builder, robot_audio_vector)
kfs = CozmoAnim.Keyframes.KeyframesEnd(builder)
name_str = builder.CreateString(self.name)
CozmoAnim.AnimClip.AnimClipStart(builder)
CozmoAnim.AnimClip.AnimClipAddName(builder, name_str)
CozmoAnim.AnimClip.AnimClipAddKeyframes(builder, kfs)
fbclip = CozmoAnim.AnimClip.AnimClipEnd(builder)
return fbclip
def serialize(entity: SpacyResponseData, builder: Builder) -> Any:
ner_offsets = list(map(lambda n: NERSpacyResponseMapper.serialize(n, builder), entity.ner))
dep_offsets = list(map(lambda d: DEPSpacyResponseMapper.serialize(d, builder), entity.dep))
SpacyResponseClass.SpacyResponseStartNerVector(builder, len(entity.ner))
for x in reversed(ner_offsets):
builder.PrependUOffsetTRelative(x)
ner_vector_offset = builder.EndVector(len(entity.ner))
SpacyResponseClass.SpacyResponseStartDepVector(builder, len(entity.dep))
for x in reversed(dep_offsets):
builder.PrependUOffsetTRelative(x)
dep_vector_offset = builder.EndVector(len(entity.dep))
SpacyResponseClass.SpacyResponseStart(builder)
SpacyResponseClass.SpacyResponseAddNer(builder, ner_vector_offset)
SpacyResponseClass.SpacyResponseAddDep(builder, dep_vector_offset)
return SpacyResponseClass.SpacyResponseEnd(builder)
def _create_schema(builder: flatbuffers.Builder,
root_object: Object.Object) -> Schema.Schema:
Schema.SchemaStartObjectsVector(builder, 1)
builder.PrependUOffsetTRelative(root_object)
objects = builder.EndVector(1)
Schema.SchemaStart(builder)
Schema.SchemaAddObjects(builder, objects)
Schema.SchemaAddRootTable(builder, root_object)
return Schema.SchemaEnd(builder)
def serialize(entity: SpacyBatchResponseData, builder: Builder) -> Any:
responses_offsets = list(map(lambda r: SpacyResponseMapper.serialize(r, builder), entity.responses))
SpacyBatchResponseClass.SpacyBatchResponseStartResponsesVector(builder, len(entity.responses))
for offset in reversed(responses_offsets):
builder.PrependUOffsetTRelative(offset)
responses_vector_offset = builder.EndVector(len(entity.responses))
SpacyBatchResponseClass.SpacyBatchResponseStart(builder)
SpacyBatchResponseClass.SpacyBatchResponseAddResponses(builder, responses_vector_offset)
return SpacyBatchResponseClass.SpacyBatchResponseEnd(builder)
def _create_object(builder: flatbuffers.Builder, name: str,
fields: Sequence[Field.Field]) -> Object.Object:
name = builder.CreateString(name)
Object.ObjectStartFieldsVector(builder, len(fields))
for field in reversed(fields):
builder.PrependUOffsetTRelative(field)
fields_vector = builder.EndVector(len(fields))
Object.ObjectStart(builder)
Object.ObjectAddName(builder, name)
Object.ObjectAddFields(builder, fields_vector)
return Object.ObjectEnd(builder)
def create_flatbuffer(self) -> Builder:
builder = Builder(1000)
name_dict = {}
player_config_offsets = [
pc.write_to_flatbuffer(builder, name_dict)
for pc in self.player_configs
]
MatchSettingsFlat.MatchSettingsStartPlayerConfigurationsVector(
builder, len(player_config_offsets))
for i in reversed(range(0, len(player_config_offsets))):
builder.PrependUOffsetTRelative(player_config_offsets[i])
player_list_offset = builder.EndVector(len(player_config_offsets))
mutator_settings_offset = self.mutators.write_to_flatbuffer(builder)
if self.game_map in game_map_dict:
upk = game_map_dict[self.game_map]
map_index = map_types.index(self.game_map)
else:
self.logger.info(
f"Did not recognize {self.game_map}, hoping it's new or a custom map!"
)
upk = self.game_map
map_index = -1
upk_offset = builder.CreateString(upk)
MatchSettingsFlat.MatchSettingsStart(builder)
MatchSettingsFlat.MatchSettingsAddPlayerConfigurations(
builder, player_list_offset)
MatchSettingsFlat.MatchSettingsAddGameMode(
builder, game_mode_types.index(self.game_mode))
MatchSettingsFlat.MatchSettingsAddGameMap(builder, map_index)
MatchSettingsFlat.MatchSettingsAddGameMapUpk(builder, upk_offset)
MatchSettingsFlat.MatchSettingsAddSkipReplays(builder,
self.skip_replays)
MatchSettingsFlat.MatchSettingsAddInstantStart(builder,
self.instant_start)
MatchSettingsFlat.MatchSettingsAddMutatorSettings(
builder, mutator_settings_offset)
MatchSettingsFlat.MatchSettingsAddExistingMatchBehavior(
builder,
index_or_zero(existing_match_behavior_types,
self.existing_match_behavior))
MatchSettingsFlat.MatchSettingsAddEnableLockstep(
builder, self.enable_lockstep)
MatchSettingsFlat.MatchSettingsAddEnableRendering(
builder, self.enable_rendering)
MatchSettingsFlat.MatchSettingsAddEnableStateSetting(
builder, self.enable_state_setting)
MatchSettingsFlat.MatchSettingsAddAutoSaveReplay(
builder, self.auto_save_replay)
ms_offset = MatchSettingsFlat.MatchSettingsEnd(builder)
builder.Finish(ms_offset)
return builder
def serialize(data: EmbeddingResponseData, builder: flatbuffers.Builder):
length = len(data.vector)
vector_offsets = list(
map(lambda r: EmbeddingElementMapper.serialize(r, builder),
data.vector))
EmbeddingResponse.EmbeddingResponseStartVectorVector(builder, length)
for offset in reversed(vector_offsets):
builder.PrependUOffsetTRelative(offset)
vector_offsets_vector = builder.EndVector(length)
EmbeddingResponse.EmbeddingResponseStart(builder)
EmbeddingResponse.EmbeddingResponseAddVector(builder,
vector_offsets_vector)
return EmbeddingResponse.EmbeddingResponseEnd(builder)
def serialize(data: MostSimilarResponseData, builder: flatbuffers.Builder):
length = len(data.similarities)
similarities_offsets = list(
map(lambda r: MostSimilarityMapper.serialize(r, builder),
data.similarities))
MostSimilarResponse.MostSimilarResponseStartSimilaritiesVector(
builder, length)
for offset in reversed(similarities_offsets):
builder.PrependUOffsetTRelative(offset)
similarities_offsets_vector = builder.EndVector(length)
MostSimilarResponse.MostSimilarResponseStart(builder)
MostSimilarResponse.MostSimilarResponseAddSimilarities(
builder, similarities_offsets_vector)
return MostSimilarResponse.MostSimilarResponseEnd(builder)
def to_fb(self, builder: flatbuffers.Builder):
clips_arr = []
for clip in self.clips:
fbclip = clip.to_fb(builder)
clips_arr.append(fbclip)
CozmoAnim.AnimClips.AnimClipsStartClipsVector(builder, len(clips_arr))
for i in reversed(range(len(clips_arr))):
builder.PrependUOffsetTRelative(clips_arr[i])
clips_vector = builder.EndVector(len(clips_arr))
CozmoAnim.AnimClips.AnimClipsStart(builder)
CozmoAnim.AnimClips.AnimClipsAddClips(builder, clips_vector)
fbclips = CozmoAnim.AnimClips.AnimClipsEnd(builder)
return fbclips
def serialize(data: GensimFastTextMostSimilarResponseData,
builder: flatbuffers.Builder):
length = len(data.responses)
responses_offsets = list(
map(lambda r: MostSimilarResponseMapper.serialize(r, builder),
data.responses))
GensimFastTextMostSimilarResponse.GensimFastTextMostSimilarResponseStartResponsesVector(
builder, length)
for offset in reversed(responses_offsets):
builder.PrependUOffsetTRelative(offset)
responses_offsets_vector = builder.EndVector(length)
GensimFastTextMostSimilarResponse.GensimFastTextMostSimilarResponseStart(
builder)
GensimFastTextMostSimilarResponse.GensimFastTextMostSimilarResponseAddResponses(
builder, responses_offsets_vector)
return GensimFastTextMostSimilarResponse.GensimFastTextMostSimilarResponseEnd(
builder)
def serialize(data: GensimFastTextEmbeddingRequestData,
builder: flatbuffers.Builder):
length = len(data.requests)
request_offsets = list(
map(lambda r: EmbeddingRequestMapper.serialize(r, builder),
data.requests))
GensimFastTextEmbeddingRequest.GensimFastTextEmbeddingRequestStartRequestsVector(
builder, length)
for offset in reversed(request_offsets):
builder.PrependUOffsetTRelative(offset)
request_offsets_vector = builder.EndVector(length)
GensimFastTextEmbeddingRequest.GensimFastTextEmbeddingRequestStart(
builder)
GensimFastTextEmbeddingRequest.GensimFastTextEmbeddingRequestAddRequests(
builder, request_offsets_vector)
return GensimFastTextEmbeddingRequest.GensimFastTextEmbeddingRequestEnd(
builder)
def _encode_vector(self, builder: flatbuffers.Builder, value: Collection,
element_type: BaseType) -> int:
if element_type == BaseType.UByte or element_type == BaseType.Byte:
return builder.CreateByteVector(value)
elements_encoded = [
self._encode_value(builder, element, element_type)
for element in value
]
element_bytesize = _number_type_by_base_type[element_type].bytewidth
vector_element_type = _number_type_by_base_type[element_type]
builder.StartVector(element_bytesize, len(elements_encoded),
element_bytesize)
for index, element in enumerate(reversed(elements_encoded)):
if vector_element_type == flatbuffers.number_types.UOffsetTFlags:
builder.PrependUOffsetTRelative(element)
else:
builder.Prepend(_number_type_by_base_type[element_type],
element)
return builder.EndVector(len(elements_encoded))
def to_fb(self, builder: flatbuffers.Builder):
head_angle_arr = []
lift_height_arr = []
record_heading_arr = []
turn_to_recorded_heading_arr = []
body_motion_arr = []
backpack_lights_arr = []
face_animation_arr = []
procedural_face_arr = []
robot_audio_arr = []
event_arr = []
for keyframe in self.keyframes:
fbkf = keyframe.to_fb(builder)
if isinstance(keyframe, AnimHeadAngle):
head_angle_arr.append(fbkf)
elif isinstance(keyframe, AnimLiftHeight):
lift_height_arr.append(fbkf)
elif isinstance(keyframe, AnimRecordHeading):
record_heading_arr.append(fbkf)
elif isinstance(keyframe, AnimTurnToRecordedHeading):
turn_to_recorded_heading_arr.append(fbkf)
elif isinstance(keyframe, AnimBodyMotion):
body_motion_arr.append(fbkf)
elif isinstance(keyframe, AnimBackpackLights):
backpack_lights_arr.append(fbkf)
elif isinstance(keyframe, AnimFaceAnimation):
face_animation_arr.append(fbkf)
elif isinstance(keyframe, AnimProceduralFace):
procedural_face_arr.append(fbkf)
elif isinstance(keyframe, AnimRobotAudio):
robot_audio_arr.append(fbkf)
elif isinstance(keyframe, AnimEvent):
event_arr.append(fbkf)
else:
raise RuntimeError("Unexpected keyframe type.")
CozmoAnim.Keyframes.KeyframesStartHeadAngleKeyFrameVector(
builder, len(head_angle_arr))
for i in reversed(range(len(head_angle_arr))):
builder.PrependUOffsetTRelative(head_angle_arr[i])
head_angle_vector = builder.EndVector(len(head_angle_arr))
CozmoAnim.Keyframes.KeyframesStartLiftHeightKeyFrameVector(
builder, len(lift_height_arr))
for i in reversed(range(len(lift_height_arr))):
builder.PrependUOffsetTRelative(lift_height_arr[i])
lift_height_vector = builder.EndVector(len(lift_height_arr))
CozmoAnim.Keyframes.KeyframesStartRecordHeadingKeyFrameVector(
builder, len(record_heading_arr))
for i in reversed(range(len(record_heading_arr))):
builder.PrependUOffsetTRelative(record_heading_arr[i])
record_heading_vector = builder.EndVector(len(record_heading_arr))
CozmoAnim.Keyframes.KeyframesStartTurnToRecordedHeadingKeyFrameVector(
builder, len(turn_to_recorded_heading_arr))
for i in reversed(range(len(turn_to_recorded_heading_arr))):
builder.PrependUOffsetTRelative(turn_to_recorded_heading_arr[i])
turn_to_recorded_heading_vector = builder.EndVector(
len(turn_to_recorded_heading_arr))
CozmoAnim.Keyframes.KeyframesStartBackpackLightsKeyFrameVector(
builder, len(backpack_lights_arr))
for i in reversed(range(len(backpack_lights_arr))):
builder.PrependUOffsetTRelative(backpack_lights_arr[i])
backpack_lights_vector = builder.EndVector(len(backpack_lights_arr))
CozmoAnim.Keyframes.KeyframesStartBodyMotionKeyFrameVector(
builder, len(body_motion_arr))
for i in reversed(range(len(body_motion_arr))):
builder.PrependUOffsetTRelative(body_motion_arr[i])
body_motion_vector = builder.EndVector(len(body_motion_arr))
CozmoAnim.Keyframes.KeyframesStartFaceAnimationKeyFrameVector(
builder, len(face_animation_arr))
for i in reversed(range(len(face_animation_arr))):
builder.PrependUOffsetTRelative(face_animation_arr[i])
face_animation_vector = builder.EndVector(len(face_animation_arr))
CozmoAnim.Keyframes.KeyframesStartProceduralFaceKeyFrameVector(
builder, len(procedural_face_arr))
for i in reversed(range(len(procedural_face_arr))):
builder.PrependUOffsetTRelative(procedural_face_arr[i])
procedural_face_vector = builder.EndVector(len(procedural_face_arr))
CozmoAnim.Keyframes.KeyframesStartRobotAudioKeyFrameVector(
builder, len(robot_audio_arr))
for i in reversed(range(len(robot_audio_arr))):
builder.PrependUOffsetTRelative(robot_audio_arr[i])
robot_audio_vector = builder.EndVector(len(robot_audio_arr))
CozmoAnim.Keyframes.KeyframesStartEventKeyFrameVector(
builder, len(event_arr))
for i in reversed(range(len(event_arr))):
builder.PrependUOffsetTRelative(event_arr[i])
event_vector = builder.EndVector(len(event_arr))
CozmoAnim.Keyframes.KeyframesStart(builder)
CozmoAnim.Keyframes.KeyframesAddHeadAngleKeyFrame(
builder, head_angle_vector)
CozmoAnim.Keyframes.KeyframesAddLiftHeightKeyFrame(
builder, lift_height_vector)
CozmoAnim.Keyframes.KeyframesAddRecordHeadingKeyFrame(
builder, record_heading_vector)
CozmoAnim.Keyframes.KeyframesAddTurnToRecordedHeadingKeyFrame(
builder, turn_to_recorded_heading_vector)
CozmoAnim.Keyframes.KeyframesAddBodyMotionKeyFrame(
builder, body_motion_vector)
CozmoAnim.Keyframes.KeyframesAddBackpackLightsKeyFrame(
builder, backpack_lights_vector)
CozmoAnim.Keyframes.KeyframesAddFaceAnimationKeyFrame(
builder, face_animation_vector)
CozmoAnim.Keyframes.KeyframesAddProceduralFaceKeyFrame(
builder, procedural_face_vector)
CozmoAnim.Keyframes.KeyframesAddRobotAudioKeyFrame(
builder, robot_audio_vector)
CozmoAnim.Keyframes.KeyframesAddEventKeyFrame(builder, event_vector)
kfs = CozmoAnim.Keyframes.KeyframesEnd(builder)
name_str = builder.CreateString(self.name)
CozmoAnim.AnimClip.AnimClipStart(builder)
CozmoAnim.AnimClip.AnimClipAddName(builder, name_str)
CozmoAnim.AnimClip.AnimClipAddKeyframes(builder, kfs)
fbclip = CozmoAnim.AnimClip.AnimClipEnd(builder)
return fbclip
def handle_meta_layer(g: minidom.Node, objs: GameObjects,
builder: flatbuffers.Builder):
for o in g.getElementsByTagName('object'):
x, y, rotation = get_pos(o)
width, height = get_dim(o)
typ = o.getAttribute('type')
if typ == 'playerwall':
x += (math.cos(math.radians(rotation)) * width / 2.0 -
math.sin(math.radians(rotation)) * height / 2.0)
y += (math.cos(math.radians(rotation)) * height / 2.0 +
math.sin(math.radians(rotation)) * width / 2.0)
FlatBuffGenerated.PlayerWall.PlayerWallStart(builder)
pos = FlatBuffGenerated.Vec2.CreateVec2(builder, x * GLOBAL_SCALE,
y * GLOBAL_SCALE)
FlatBuffGenerated.PlayerWall.PlayerWallAddPosition(builder, pos)
size = FlatBuffGenerated.Vec2.CreateVec2(
builder, width / 2.0 * GLOBAL_SCALE,
height / 2.0 * GLOBAL_SCALE)
FlatBuffGenerated.PlayerWall.PlayerWallAddSize(builder, size)
FlatBuffGenerated.PlayerWall.PlayerWallAddRotation(
builder, rotation)
pwall = FlatBuffGenerated.PlayerWall.PlayerWallEnd(builder)
objs.playerwalls.append(pwall)
elif typ == 'waypoint':
name = o.getAttribute('name')
neighbors = []
isspawn = False
isplayerspawn = False
assertCircleness(o)
x += (math.cos(math.radians(rotation)) * width / 2.0 -
math.sin(math.radians(rotation)) * height / 2.0)
y += (math.cos(math.radians(rotation)) * height / 2.0 +
math.sin(math.radians(rotation)) * width / 2.0)
radius = (width / 2.0) * GLOBAL_SCALE
for prop in o.getElementsByTagName('property'):
prop_name = prop.getAttribute('name')
value = prop.getAttribute('value')
if prop_name == 'wps':
neighbors = value.split(',')
elif prop_name == 'isspawn':
isspawn = (value == 'true')
elif prop_name == 'isplayerspawn':
isplayerspawn = (value == 'true')
else:
print("WARNING: Unknown property {}".format(prop_name))
neighbors = list(filter(lambda x: x.strip() != "", neighbors))
if len(neighbors) < 2 and not isplayerspawn and not isspawn:
print("WARNING: waypoint {} has < 2 neighbors".format(name))
name = builder.CreateString(name)
neighOff = [builder.CreateString(x) for x in neighbors]
FlatBuffGenerated.NavPoint.NavPointStartNeighborsVector(
builder, len(neighOff))
for b in neighOff:
builder.PrependUOffsetTRelative(b)
neighs = builder.EndVector(len(neighOff))
FlatBuffGenerated.NavPoint.NavPointStart(builder)
pos = FlatBuffGenerated.Vec2.CreateVec2(builder, x * GLOBAL_SCALE,
y * GLOBAL_SCALE)
FlatBuffGenerated.NavPoint.NavPointAddPosition(builder, pos)
FlatBuffGenerated.NavPoint.NavPointAddRadius(builder, radius)
FlatBuffGenerated.NavPoint.NavPointAddName(builder, name)
FlatBuffGenerated.NavPoint.NavPointAddNeighbors(builder, neighs)
FlatBuffGenerated.NavPoint.NavPointAddIsspawn(builder, isspawn)
FlatBuffGenerated.NavPoint.NavPointAddIsplayerspawn(
builder, isplayerspawn)
objs.navpoints.append(
FlatBuffGenerated.NavPoint.NavPointEnd(builder))