def _merge_field( datasource_series: List[Tuple[str, pd.Series]], new_index, log: structlog.BoundLoggerBase ): log.info("Working field") field_out = None field_provenance = pd.Series(index=new_index, dtype="object") # Go through the data sources, starting with the highest priority. for datasource_name, datasource_field_in in reversed(datasource_series): log_datasource = log.bind(dataset_name=datasource_name) if field_out is None: # Copy all values from the highest priority input to the output field_provenance.loc[pd.notna(datasource_field_in)] = datasource_name field_out = datasource_field_in else: field_out_has_ts = field_out.groupby(level=[CommonFields.FIPS], sort=False).transform( lambda x: x.notna().any() ) copy_field_in = (~field_out_has_ts) & pd.notna(datasource_field_in) # Copy from datasource_field_in only on rows where all rows of field_out with that FIPS are NaN. field_provenance.loc[copy_field_in] = datasource_name field_out = field_out.where(~copy_field_in, datasource_field_in) dups = field_out.index.duplicated(keep=False) if dups.any(): log_datasource.error("Found duplicates in index") raise ValueError() # This is bad, somehow the input /still/ has duplicates return field_out, field_provenance
def _map_columns(self, df: pd.DataFrame, log: structlog.BoundLoggerBase) -> pd.DataFrame: """Given a DataFrame with `Fields` columns, return county data with `CommonFields` columns.""" df = df.merge( self.geo_fields_to_common_fields, left_on=[Fields.GEO_TYPE, Fields.GEO_VALUE], suffixes=(False, False), how="left", right_index=True, ) no_match_mask = df[CommonFields.FIPS].isna() if no_match_mask.sum() > 0: log.warning( "Dropping rows that did not merge by geo_value", geo_value_count=df.loc[no_match_mask].groupby( Fields.GEO_VALUE).size().to_dict(), ) df = df.loc[~no_match_mask, :] df = df.drop(columns=[ f for f in Fields if f in df.columns and f.common_field is None ]).rename(columns={ f: f.common_field for f in Fields if f.common_field is not None }) return df
async def on_rejoin_game(*, log: Logger, ctx: ServerCtx, conn_id: str, msg_id: int, payload: dict): game_id = payload['game_id'] player = payload['player'] player_nonce = payload['player_nonce'] game_id_bytes = uuid.UUID(game_id).bytes session_state, game_id = await ctx.redis_store.read_session(conn_id) if session_state != SessionState.NEED_JOIN: log.msg('unexpected session state', session_state=session_state.value) await ctx.send_fatal( conn_id, wire.ServerMsgType.ILLEGAL_MSG, error='session is not awaiting join', err_msg_id=msg_id, ) return meta = await ctx.redis_store.read_game_meta(game_id_bytes) if not await validate_rejoin_request( ctx, log, player=player, declared_nonce=player_nonce, meta=meta): return if meta.state == GameState.RUNNING: _, game = await ctx.redis_store.read_game(game_id_bytes) try: await ctx.send( conn_id, wire.ServerMsgType.MOVE_PENDING, player=game.player, last_move=meta.get_last_move_json(), ) except WebsocketConnectionLost: log.msg('connection lost') meta = meta.with_conn_id(player, None) await asyncio.gather( ctx.redis_store.delete_session(conn_id), ctx.redis_store.update_game(game_id_bytes, meta=meta), ) return await asyncio.gather( ctx.redis_store.put_session(conn_id, state=SessionState.RUNNING, game_id=game_id), ctx.redis_store.update_game(game_id_bytes, meta=meta.with_conn_id( player, conn_id)), )
async def on_new_game(*, log: Logger, ctx: ServerCtx, conn_id: str, msg_id: int, payload: dict): player: Player = payload['player'] squares: int = payload['squares_per_row'] target_len: int = payload['run_to_win'] # TODO: handle validation of relative values of params game = GameModel(squares=squares, target_len=target_len) meta = GameMeta( state=GameState.JOIN_PENDING, player_nonces=(uuid.uuid4(), uuid.uuid4()), conn_ids=(conn_id, None) if player == 1 else (None, conn_id), last_move=None, ) session_state, _ = await ctx.redis_store.read_session(conn_id) if session_state != SessionState.NEED_JOIN: log.msg('unexpected session state', session_state=session_state.value) await ctx.send_fatal( conn_id, wire.ServerMsgType.ILLEGAL_MSG, error='session is not awaiting join', err_msg_id=msg_id, ) return game_key = await ctx.redis_store.put_game(game=game, meta=meta) await ctx.redis_store.put_session(conn_id, SessionState.NEED_JOIN_ACK, game_key.bytes) try: await ctx.send( conn_id, wire.ServerMsgType.GAME_JOINED, player=player, squares_per_row=squares, run_to_win=target_len, game_id=str(game_key), # TODO player_nonce=str(meta.get_player_nonce(player)), ) except WebsocketConnectionLost: log.msg('connection lost') await asyncio.gather( ctx.redis_store.delete_game(game_key.bytes), ctx.redis_store.delete_session(conn_id), )
async def on_new_move(*, log: Logger, ctx: ServerCtx, conn_id: str, msg_id: int, payload: dict) -> None: session_state, game_id = await ctx.redis_store.read_session(conn_id) if session_state != SessionState.RUNNING: # TODO: disconnect await ctx.send_fatal(conn_id, wire.ServerMsgType.ILLEGAL_MSG, error='', err_msg_id=msg_id) return if game_id is None: raise RuntimeError( 'No game associated with session in running state') meta, (_, game) = await asyncio.gather( ctx.redis_store.read_game_meta(game_id), ctx.redis_store.read_game(game_id)) player = meta.get_player_for_conn_id(conn_id) if player is None: log.msg('Connection cleared from game state') await ctx.ws_manager.close(conn_id) return try: coords = payload['x'], payload['y'] move = Move(player=player, coords=coords) game.apply_move(move) except IllegalMoveException as exc: await ctx.send_fatal(conn_id, wire.ServerMsgType.ILLEGAL_MOVE, error=str(exc)) # TODO: update game state return meta = meta.with_last_move(move) if game.status() != GameStatus.Ongoing: meta = meta.with_state(GameState.COMPLETED) await ctx.redis_store.update_game(game_id, meta=meta, game=game) await broadcast_game_state(ctx, meta=meta, game=game)
async def validate_rejoin_request(ctx: ServerCtx, log: Logger, player: Player, declared_nonce: str, meta: GameMeta) -> bool: if meta.get_player_nonce(player) != declared_nonce: log.msg('incorrect nonce') return False prior_conn_id = meta.get_conn_id(player) if prior_conn_id is not None: # Clean up any connection which failed to ack its join before this request came in # FIXME: I don't really like this racing approach log.msg('tearing down prior connection', prior_conn_id=prior_conn_id) await asyncio.gather( ctx.redis_store.delete_session(prior_conn_id), ctx.ws_manager.close(prior_conn_id), ) return True
def log_transaction(log: BoundLoggerBase, description: str, details: Dict[Any, Any]) -> Generator: bound_log = log.bind(description=description, **details) try: bound_log.debug("Entered") yield except: # noqa bound_log.critical("Failed", exc_info=True) raise bound_log.debug("Exited")
async def validate_join_request(ctx: ServerCtx, log: Logger, player: Player, meta: GameMeta) -> bool: if meta.state != GameState.JOIN_PENDING: log.msg('game state is not pending join', game_state=meta.state.value) return False prior_conn_id = meta.get_conn_id(player) if prior_conn_id is None: return True prior_session_state, _ = await ctx.redis_store.read_session(prior_conn_id) if prior_session_state == SessionState.NEED_JOIN_ACK: # Clean up any connection which failed to ack its join before this request came in # FIXME: I don't really like this racing approach log.msg('tearing down prior connection', prior_conn_id=prior_conn_id) await asyncio.gather( ctx.redis_store.delete_session(prior_conn_id), ctx.ws_manager.close(prior_conn_id), ) return True log.msg( 'prior connection already joined', prior_conn_id=prior_conn_id, prior_session_state=prior_session_state.value, ) return False
async def on_ack_game_joined(*, log: Logger, ctx: ServerCtx, conn_id: str, msg_id: int, payload: dict): session_state, game_id = await ctx.redis_store.read_session(conn_id) if session_state != SessionState.NEED_JOIN_ACK: log.msg('unexpected session state', session_state=session_state.value) await ctx.send_fatal( conn_id, wire.ServerMsgType.ILLEGAL_MSG, error='unexpected ack_game_joined', err_msg_id=msg_id, ) await ctx.redis_store.delete_session(conn_id) return assert game_id is not None meta = await ctx.redis_store.read_game_meta(game_id) player = meta.get_player_for_conn_id(conn_id) if player is None: log.msg('connection cleared from game') # FIXME: shouldn't happen? await ctx.send_fatal( conn_id, wire.ServerMsgType.ILLEGAL_MSG, error='game cleared', err_msg_id=msg_id, ) await ctx.redis_store.delete_session(conn_id) return await ctx.redis_store.put_session(conn_id, state=SessionState.RUNNING, game_id=game_id) other_conn = meta.get_conn_id(get_opponent(player)) if other_conn is None: all_joined = False else: other_state, _ = await ctx.redis_store.read_session(other_conn) all_joined = other_state == SessionState.RUNNING if all_joined: log.msg('game fully joined') meta = meta.with_state(GameState.RUNNING) await ctx.redis_store.update_game(game_id, meta=meta) _, game = await ctx.redis_store.read_game(game_id) await broadcast_game_state(ctx, meta, game)
def filter_and_smooth_input_data( df: pd.DataFrame, region: pipeline.Region, include_deaths: bool, figure_collector: Optional[list], log: structlog.BoundLoggerBase, ) -> pd.DataFrame: """Do Filtering Here Before it Gets to the Inference Engine""" MIN_CUMULATIVE_COUNTS = dict(cases=20, deaths=10) MIN_INCIDENT_COUNTS = dict(cases=5, deaths=5) dates = df.index # Apply Business Logic To Filter Raw Data for column in ["cases", "deaths"]: requirements = [ # All Must Be True df[column].count() > InferRtConstants.MIN_TIMESERIES_LENGTH, df[column].sum() > MIN_CUMULATIVE_COUNTS[column], df[column].max() > MIN_INCIDENT_COUNTS[column], ] # Now Apply Input Outlier Detection and Smoothing filtered = utils.replace_outliers(df[column], log=rt_log.new(column=column)) # TODO find way to indicate which points filtered in figure below assert len(filtered) == len(df[column]) smoothed = filtered.rolling( InferRtConstants.COUNT_SMOOTHING_WINDOW_SIZE, win_type="gaussian", min_periods=InferRtConstants.COUNT_SMOOTHING_KERNEL_STD, center=True, ).mean(std=InferRtConstants.COUNT_SMOOTHING_KERNEL_STD) # TODO: Only start once non-zero to maintain backwards compatibility? # Check if the Post Smoothed Meets the Requirements requirements.append(smoothed.max() > MIN_INCIDENT_COUNTS[column]) # Check include_deaths Flag if column == "deaths" and not include_deaths: requirements.append(False) else: requirements.append(True) if all(requirements): if column == "cases": fig = plt.figure(figsize=(10, 6)) ax = fig.add_subplot(111) # plt.axes ax.set_yscale("log") chart_min = max(0.1, smoothed.min()) ax.set_ylim((chart_min, df[column].max())) plt.scatter( dates[-len(df[column]) :], df[column], alpha=0.3, label=f"Smoothing of: {column}", ) plt.plot(dates[-len(df[column]) :], smoothed) plt.grid(True, which="both") plt.xticks(rotation=30) plt.xlim(min(dates[-len(df[column]) :]), max(dates) + timedelta(days=2)) if not figure_collector: plot_path = pyseir.utils.get_run_artifact_path( region, RunArtifact.RT_SMOOTHING_REPORT ) plt.savefig(plot_path, bbox_inches="tight") plt.close(fig) else: figure_collector["1_smoothed_cases"] = fig df[column] = smoothed else: df = df.drop(columns=column, inplace=False) log.info("Dropping:", columns=column, requirements=requirements) return df
async def on_join_game(*, log: Logger, ctx: ServerCtx, conn_id: str, msg_id: int, payload: dict): game_id: str = payload['game_id'] player: Player = payload['player'] log = log.bind(game_id=game_id, player=player) game_id_bytes = uuid.UUID(game_id).bytes session_state, _ = await ctx.redis_store.read_session(conn_id) if session_state != SessionState.NEED_JOIN: log.msg('unexpected session state', session_state=session_state.value) await ctx.send_fatal( conn_id, wire.ServerMsgType.ILLEGAL_MSG, error='session is not awaiting join', err_msg_id=msg_id, ) return meta = await ctx.redis_store.read_game_meta(game_id_bytes) if not await validate_join_request( ctx, log=log, player=player, meta=meta): log.msg('player already claimed') await ctx.send_fatal( conn_id, wire.ServerMsgType.ILLEGAL_MSG, error='player has already been claimed', err_msg_id=msg_id, ) return meta = meta.with_conn_id(player, conn_id) # TODO: validate state transitions? await asyncio.gather( ctx.redis_store.update_game(game_id_bytes, meta=meta), ctx.redis_store.put_session(conn_id, SessionState.NEED_JOIN_ACK, game_id_bytes), ) # XXX: org? _, game = await ctx.redis_store.read_game(game_id_bytes) try: await ctx.send( conn_id, wire.ServerMsgType.GAME_JOINED, game_id=game_id, player=player, player_nonce=str(meta.get_player_nonce(player)), squares_per_row=game.squares, run_to_win=game.target_len, ) except WebsocketConnectionLost: log.msg('connection lost') # Roll back game updates... meta = meta.with_state(GameState.JOIN_PENDING).with_conn_id( player, None) await asyncio.gather( ctx.redis_store.delete_session(conn_id), ctx.redis_store.update_game(game_id_bytes, meta=meta), )