def _merge_field(
datasource_series: List[Tuple[str, pd.Series]], new_index, log: structlog.BoundLoggerBase
):
log.info("Working field")
field_out = None
field_provenance = pd.Series(index=new_index, dtype="object")
# Go through the data sources, starting with the highest priority.
for datasource_name, datasource_field_in in reversed(datasource_series):
log_datasource = log.bind(dataset_name=datasource_name)
if field_out is None:
# Copy all values from the highest priority input to the output
field_provenance.loc[pd.notna(datasource_field_in)] = datasource_name
field_out = datasource_field_in
else:
field_out_has_ts = field_out.groupby(level=[CommonFields.FIPS], sort=False).transform(
lambda x: x.notna().any()
)
copy_field_in = (~field_out_has_ts) & pd.notna(datasource_field_in)
# Copy from datasource_field_in only on rows where all rows of field_out with that FIPS are NaN.
field_provenance.loc[copy_field_in] = datasource_name
field_out = field_out.where(~copy_field_in, datasource_field_in)
dups = field_out.index.duplicated(keep=False)
if dups.any():
log_datasource.error("Found duplicates in index")
raise ValueError() # This is bad, somehow the input /still/ has duplicates
return field_out, field_provenance
def _map_columns(self, df: pd.DataFrame,
log: structlog.BoundLoggerBase) -> pd.DataFrame:
"""Given a DataFrame with `Fields` columns, return county data with `CommonFields` columns."""
df = df.merge(
self.geo_fields_to_common_fields,
left_on=[Fields.GEO_TYPE, Fields.GEO_VALUE],
suffixes=(False, False),
how="left",
right_index=True,
)
no_match_mask = df[CommonFields.FIPS].isna()
if no_match_mask.sum() > 0:
log.warning(
"Dropping rows that did not merge by geo_value",
geo_value_count=df.loc[no_match_mask].groupby(
Fields.GEO_VALUE).size().to_dict(),
)
df = df.loc[~no_match_mask, :]
df = df.drop(columns=[
f for f in Fields if f in df.columns and f.common_field is None
]).rename(columns={
f: f.common_field
for f in Fields if f.common_field is not None
})
return df
async def on_rejoin_game(*, log: Logger, ctx: ServerCtx, conn_id: str,
msg_id: int, payload: dict):
game_id = payload['game_id']
player = payload['player']
player_nonce = payload['player_nonce']
game_id_bytes = uuid.UUID(game_id).bytes
session_state, game_id = await ctx.redis_store.read_session(conn_id)
if session_state != SessionState.NEED_JOIN:
log.msg('unexpected session state',
session_state=session_state.value)
await ctx.send_fatal(
conn_id,
wire.ServerMsgType.ILLEGAL_MSG,
error='session is not awaiting join',
err_msg_id=msg_id,
)
return
meta = await ctx.redis_store.read_game_meta(game_id_bytes)
if not await validate_rejoin_request(
ctx, log, player=player, declared_nonce=player_nonce,
meta=meta):
return
if meta.state == GameState.RUNNING:
_, game = await ctx.redis_store.read_game(game_id_bytes)
try:
await ctx.send(
conn_id,
wire.ServerMsgType.MOVE_PENDING,
player=game.player,
last_move=meta.get_last_move_json(),
)
except WebsocketConnectionLost:
log.msg('connection lost')
meta = meta.with_conn_id(player, None)
await asyncio.gather(
ctx.redis_store.delete_session(conn_id),
ctx.redis_store.update_game(game_id_bytes, meta=meta),
)
return
await asyncio.gather(
ctx.redis_store.put_session(conn_id,
state=SessionState.RUNNING,
game_id=game_id),
ctx.redis_store.update_game(game_id_bytes,
meta=meta.with_conn_id(
player, conn_id)),
)
async def on_new_game(*, log: Logger, ctx: ServerCtx, conn_id: str,
msg_id: int, payload: dict):
player: Player = payload['player']
squares: int = payload['squares_per_row']
target_len: int = payload['run_to_win']
# TODO: handle validation of relative values of params
game = GameModel(squares=squares, target_len=target_len)
meta = GameMeta(
state=GameState.JOIN_PENDING,
player_nonces=(uuid.uuid4(), uuid.uuid4()),
conn_ids=(conn_id, None) if player == 1 else (None, conn_id),
last_move=None,
)
session_state, _ = await ctx.redis_store.read_session(conn_id)
if session_state != SessionState.NEED_JOIN:
log.msg('unexpected session state',
session_state=session_state.value)
await ctx.send_fatal(
conn_id,
wire.ServerMsgType.ILLEGAL_MSG,
error='session is not awaiting join',
err_msg_id=msg_id,
)
return
game_key = await ctx.redis_store.put_game(game=game, meta=meta)
await ctx.redis_store.put_session(conn_id, SessionState.NEED_JOIN_ACK,
game_key.bytes)
try:
await ctx.send(
conn_id,
wire.ServerMsgType.GAME_JOINED,
player=player,
squares_per_row=squares,
run_to_win=target_len,
game_id=str(game_key), # TODO
player_nonce=str(meta.get_player_nonce(player)),
)
except WebsocketConnectionLost:
log.msg('connection lost')
await asyncio.gather(
ctx.redis_store.delete_game(game_key.bytes),
ctx.redis_store.delete_session(conn_id),
)
async def on_new_move(*, log: Logger, ctx: ServerCtx, conn_id: str,
msg_id: int, payload: dict) -> None:
session_state, game_id = await ctx.redis_store.read_session(conn_id)
if session_state != SessionState.RUNNING:
# TODO: disconnect
await ctx.send_fatal(conn_id,
wire.ServerMsgType.ILLEGAL_MSG,
error='',
err_msg_id=msg_id)
return
if game_id is None:
raise RuntimeError(
'No game associated with session in running state')
meta, (_, game) = await asyncio.gather(
ctx.redis_store.read_game_meta(game_id),
ctx.redis_store.read_game(game_id))
player = meta.get_player_for_conn_id(conn_id)
if player is None:
log.msg('Connection cleared from game state')
await ctx.ws_manager.close(conn_id)
return
try:
coords = payload['x'], payload['y']
move = Move(player=player, coords=coords)
game.apply_move(move)
except IllegalMoveException as exc:
await ctx.send_fatal(conn_id,
wire.ServerMsgType.ILLEGAL_MOVE,
error=str(exc))
# TODO: update game state
return
meta = meta.with_last_move(move)
if game.status() != GameStatus.Ongoing:
meta = meta.with_state(GameState.COMPLETED)
await ctx.redis_store.update_game(game_id, meta=meta, game=game)
await broadcast_game_state(ctx, meta=meta, game=game)
async def validate_rejoin_request(ctx: ServerCtx, log: Logger, player: Player,
declared_nonce: str, meta: GameMeta) -> bool:
if meta.get_player_nonce(player) != declared_nonce:
log.msg('incorrect nonce')
return False
prior_conn_id = meta.get_conn_id(player)
if prior_conn_id is not None:
# Clean up any connection which failed to ack its join before this request came in
# FIXME: I don't really like this racing approach
log.msg('tearing down prior connection', prior_conn_id=prior_conn_id)
await asyncio.gather(
ctx.redis_store.delete_session(prior_conn_id),
ctx.ws_manager.close(prior_conn_id),
)
return True
def log_transaction(log: BoundLoggerBase, description: str,
details: Dict[Any, Any]) -> Generator:
bound_log = log.bind(description=description, **details)
try:
bound_log.debug("Entered")
yield
except: # noqa
bound_log.critical("Failed", exc_info=True)
raise
bound_log.debug("Exited")
async def validate_join_request(ctx: ServerCtx, log: Logger, player: Player,
meta: GameMeta) -> bool:
if meta.state != GameState.JOIN_PENDING:
log.msg('game state is not pending join', game_state=meta.state.value)
return False
prior_conn_id = meta.get_conn_id(player)
if prior_conn_id is None:
return True
prior_session_state, _ = await ctx.redis_store.read_session(prior_conn_id)
if prior_session_state == SessionState.NEED_JOIN_ACK:
# Clean up any connection which failed to ack its join before this request came in
# FIXME: I don't really like this racing approach
log.msg('tearing down prior connection', prior_conn_id=prior_conn_id)
await asyncio.gather(
ctx.redis_store.delete_session(prior_conn_id),
ctx.ws_manager.close(prior_conn_id),
)
return True
log.msg(
'prior connection already joined',
prior_conn_id=prior_conn_id,
prior_session_state=prior_session_state.value,
)
return False
async def on_ack_game_joined(*, log: Logger, ctx: ServerCtx, conn_id: str,
msg_id: int, payload: dict):
session_state, game_id = await ctx.redis_store.read_session(conn_id)
if session_state != SessionState.NEED_JOIN_ACK:
log.msg('unexpected session state',
session_state=session_state.value)
await ctx.send_fatal(
conn_id,
wire.ServerMsgType.ILLEGAL_MSG,
error='unexpected ack_game_joined',
err_msg_id=msg_id,
)
await ctx.redis_store.delete_session(conn_id)
return
assert game_id is not None
meta = await ctx.redis_store.read_game_meta(game_id)
player = meta.get_player_for_conn_id(conn_id)
if player is None:
log.msg('connection cleared from game')
# FIXME: shouldn't happen?
await ctx.send_fatal(
conn_id,
wire.ServerMsgType.ILLEGAL_MSG,
error='game cleared',
err_msg_id=msg_id,
)
await ctx.redis_store.delete_session(conn_id)
return
await ctx.redis_store.put_session(conn_id,
state=SessionState.RUNNING,
game_id=game_id)
other_conn = meta.get_conn_id(get_opponent(player))
if other_conn is None:
all_joined = False
else:
other_state, _ = await ctx.redis_store.read_session(other_conn)
all_joined = other_state == SessionState.RUNNING
if all_joined:
log.msg('game fully joined')
meta = meta.with_state(GameState.RUNNING)
await ctx.redis_store.update_game(game_id, meta=meta)
_, game = await ctx.redis_store.read_game(game_id)
await broadcast_game_state(ctx, meta, game)
def filter_and_smooth_input_data(
df: pd.DataFrame,
region: pipeline.Region,
include_deaths: bool,
figure_collector: Optional[list],
log: structlog.BoundLoggerBase,
) -> pd.DataFrame:
"""Do Filtering Here Before it Gets to the Inference Engine"""
MIN_CUMULATIVE_COUNTS = dict(cases=20, deaths=10)
MIN_INCIDENT_COUNTS = dict(cases=5, deaths=5)
dates = df.index
# Apply Business Logic To Filter Raw Data
for column in ["cases", "deaths"]:
requirements = [ # All Must Be True
df[column].count() > InferRtConstants.MIN_TIMESERIES_LENGTH,
df[column].sum() > MIN_CUMULATIVE_COUNTS[column],
df[column].max() > MIN_INCIDENT_COUNTS[column],
]
# Now Apply Input Outlier Detection and Smoothing
filtered = utils.replace_outliers(df[column], log=rt_log.new(column=column))
# TODO find way to indicate which points filtered in figure below
assert len(filtered) == len(df[column])
smoothed = filtered.rolling(
InferRtConstants.COUNT_SMOOTHING_WINDOW_SIZE,
win_type="gaussian",
min_periods=InferRtConstants.COUNT_SMOOTHING_KERNEL_STD,
center=True,
).mean(std=InferRtConstants.COUNT_SMOOTHING_KERNEL_STD)
# TODO: Only start once non-zero to maintain backwards compatibility?
# Check if the Post Smoothed Meets the Requirements
requirements.append(smoothed.max() > MIN_INCIDENT_COUNTS[column])
# Check include_deaths Flag
if column == "deaths" and not include_deaths:
requirements.append(False)
else:
requirements.append(True)
if all(requirements):
if column == "cases":
fig = plt.figure(figsize=(10, 6))
ax = fig.add_subplot(111) # plt.axes
ax.set_yscale("log")
chart_min = max(0.1, smoothed.min())
ax.set_ylim((chart_min, df[column].max()))
plt.scatter(
dates[-len(df[column]) :],
df[column],
alpha=0.3,
label=f"Smoothing of: {column}",
)
plt.plot(dates[-len(df[column]) :], smoothed)
plt.grid(True, which="both")
plt.xticks(rotation=30)
plt.xlim(min(dates[-len(df[column]) :]), max(dates) + timedelta(days=2))
if not figure_collector:
plot_path = pyseir.utils.get_run_artifact_path(
region, RunArtifact.RT_SMOOTHING_REPORT
)
plt.savefig(plot_path, bbox_inches="tight")
plt.close(fig)
else:
figure_collector["1_smoothed_cases"] = fig
df[column] = smoothed
else:
df = df.drop(columns=column, inplace=False)
log.info("Dropping:", columns=column, requirements=requirements)
return df
async def on_join_game(*, log: Logger, ctx: ServerCtx, conn_id: str,
msg_id: int, payload: dict):
game_id: str = payload['game_id']
player: Player = payload['player']
log = log.bind(game_id=game_id, player=player)
game_id_bytes = uuid.UUID(game_id).bytes
session_state, _ = await ctx.redis_store.read_session(conn_id)
if session_state != SessionState.NEED_JOIN:
log.msg('unexpected session state',
session_state=session_state.value)
await ctx.send_fatal(
conn_id,
wire.ServerMsgType.ILLEGAL_MSG,
error='session is not awaiting join',
err_msg_id=msg_id,
)
return
meta = await ctx.redis_store.read_game_meta(game_id_bytes)
if not await validate_join_request(
ctx, log=log, player=player, meta=meta):
log.msg('player already claimed')
await ctx.send_fatal(
conn_id,
wire.ServerMsgType.ILLEGAL_MSG,
error='player has already been claimed',
err_msg_id=msg_id,
)
return
meta = meta.with_conn_id(player, conn_id)
# TODO: validate state transitions?
await asyncio.gather(
ctx.redis_store.update_game(game_id_bytes, meta=meta),
ctx.redis_store.put_session(conn_id, SessionState.NEED_JOIN_ACK,
game_id_bytes),
)
# XXX: org?
_, game = await ctx.redis_store.read_game(game_id_bytes)
try:
await ctx.send(
conn_id,
wire.ServerMsgType.GAME_JOINED,
game_id=game_id,
player=player,
player_nonce=str(meta.get_player_nonce(player)),
squares_per_row=game.squares,
run_to_win=game.target_len,
)
except WebsocketConnectionLost:
log.msg('connection lost')
# Roll back game updates...
meta = meta.with_state(GameState.JOIN_PENDING).with_conn_id(
player, None)
await asyncio.gather(
ctx.redis_store.delete_session(conn_id),
ctx.redis_store.update_game(game_id_bytes, meta=meta),
)