async def _delete(self, event: ChangeEvent): get_current_sql = "SELECT * FROM {} WHERE {} FOR UPDATE" delete_sql = "DELETE FROM {} WHERE {}" client = await self.client.acquire() try: for i in event.values: if "values" not in i.keys(): raise Exception( "update event value {} error lsn{}.".format( i, event.lsn.encode())) if not self._check_lock(event.table, i["values"]): raise Exception("need lock_key {} error lsn{}.".format( self.tables[event.table], event.lsn.encode())) where = self._parser_where(self.tables[event.table], i["values"]) await client.begin() async with client.cursor() as cur: await cur.execute( get_current_sql.format(event.table, where)) current = await cur.fetchone() if not current: # 数据已经删除 await client.rollback() continue await cur.execute(delete_sql.format(event.table, where)) await client.commit() if current != i["values"]: # 需要删除数据 # 走到这里说明有数据,切和 event 的不一致。 logger.error( "error data insert in zone_id {} table {} data {}". format(self.zone_id, event.table, current)) finally: await client.rollback() self.client.release(client)
async def _insert(self, event: ChangeEvent): get_current_sql = "SELECT * FROM {} WHERE {} FOR UPDATE" insert_sql = "INSERT INTO {} ({}) VALUES ({})" client = await self.client.acquire() try: for i in event.values: if "values" not in i.keys(): raise Exception( "update event value {} error lsn{}.".format( i, event.lsn.encode())) if not self._check_lock(event.table, i["values"]): raise Exception("need lock_key {} error lsn{}.".format( self.tables[event.table], event.lsn.encode())) where = self._parser_where(self.tables[event.table], i["values"]) await client.begin() async with client.cursor() as cur: await cur.execute( get_current_sql.format(event.table, where)) current = await cur.fetchone() if current == i["values"]: # 数据已经插入。 await client.rollback() continue if not current: # 正常插入数据 cl = ", ".join( ["`{}`".format(c) for c in i["values"].keys()]) va = ", ".join( ["'{}'".format(c) for c in i["values"].values()]) await cur.execute( insert_sql.format(event.table, cl, va)) await client.commit() # 走到这里说明有数据,切和 event 的不一致。 belong_zone_id = self._get_belong_zone_id( event.table, i["values"]) if belong_zone_id == self.zone_id: # 自己的修改 await client.rollback() continue source_zone_id, _, data_version, _ = self._parser_pidal_c( i["values"]["pidal_c"]) if belong_zone_id != source_zone_id: logger.error( "error data insert in zone_id {} table {} data {}" .format(event.source_zone_id, event.table, i["before_values"])) await client.rollback() continue # 走到这里说明来源的数据是对的,但是当前的数据不知道什么原因保留现场,不自改。 logger.error( "error data insert in zone_id {} table {} data {}". format(self.zone_id, event.table, current)) await client.rollback() finally: await client.rollback() self.client.release(client)
def _player_cast_request(self, player_cast_message, endpoint): player = self._check_player_for_event(endpoint) if False: #Check that the skill exists logger.error("Unexpected skill from player '{}'".format( player.get_character())) self._output_queue.put(OutputPack("", endpoint)) return control = player.get_control() if control: control.cast(player_cast_message.skill_id)
def _player_movement_request(self, player_movement_message, endpoint): player = self._check_player_for_event(endpoint) if not Direction.is_orthogonal(player_movement_message.direction): logger.error("Unexpected movement value from player '{}'".format( player.get_character())) self._output_queue.put(OutputPack("", endpoint)) return control = player.get_control() if control: control.move(player_movement_message.direction)
def post(self, request) -> HttpResponse: """ Handle post request to write chunks from incoming ResumableJS requests If one of the requests notices all the chunks have been fully written it will write the full file to <self.FILE_UPLOAD_DIR>/<user.plant.name_key>/<uploaded_file_name> """ total_chunks = int(request.POST.get('resumableTotalChunks')) chunk_num = int(request.POST.get('resumableChunkNumber', 1)) uploaded_file_name = request.POST.get('resumableFilename') resumable_identifier = request.POST.get('resumableIdentifier', 'error') chunk_data = request.FILES['file'] user = request.user plant_name_key = user.plant.name_key # Get - tmp file dir path for the ResumableJS upload chunks # - target file path target_file_path = self.target_file_path(plant_name_key, uploaded_file_name) chunk_dir = self.FILE_UPLOAD_DIR / plant_name_key / resumable_identifier # Save this chunk data using a lock file self._save_chunk_data(uploaded_file_name, chunk_num, chunk_data, chunk_dir) # Check if the all chunk files created in all Resumable JS Requests all_chunk_paths = self._all_chunk_paths(uploaded_file_name, chunk_dir, total_chunks) all_chunks_exists = all([p.exists() for p in all_chunk_paths]) if all_chunks_exists: # Make sure all files are finished writing, but do not wait forever tried = 0 while self._all_chunks_not_written(chunk_dir, total_chunks): tried += 1 if tried >= 5: error_msg = f'Error uploading files with temp_dir: {chunk_dir!r}' logger.error(f'[{self.LOG_PREFIX}] {error_msg}') return HttpResponseServerError(error_msg) time.sleep(1) # If all chunks writen create full file and remove chunk dir/files self._create_full_file_from_chunks(target_file_path, all_chunk_paths, chunk_dir) logger.info( f'[{self.LOG_PREFIX}] User {user!r} successfully created ' f'file {target_file_path}') return HttpResponse(status=200)
async def send(p: AIOKafkaProducer, topic: str, data: bytes, retried=False) -> None: try: future = await p.send(topic, data) await future except KafkaTimeoutError as e: logger.error("kafka producer send failed", exc_info=e) # In case of timeout, we retry once if not retried: await send(p, topic, data, retried=True) else: raise e
def _check_player_for_event(self, endpoint): player = self._room.get_player_with_endpoint(endpoint) if not player: logger.error("Received event from unknown player") self._output_queue.put(OutputPack("", endpoint)) return if not self._arena_enabled: logger.error( "Received player event from '{}' without available arena". format(player.get_character())) self._output_queue.put(OutputPack("", endpoint)) return return player
async def get(self, url, retries) -> ClientResponse: try: start = time.perf_counter() response = await self.session.get(url) total_time = time.perf_counter() - start text = await response.text() if response.status < 400: return ClientResponse(response.status, total_time, text, None) else: return ClientResponse(response.status, total_time, None, response.reason) except ClientConnectionError as e: if retries > 0: logger.error(f"Request failed while GET on url: {url}. Retrying") await self.get(self.session, retries - 1) else: raise e
def process_requests(self): while self._active: input_pack = self._input_queue.get() if input_pack.message: if isinstance(input_pack.message, Message.Version): self._info_server_request(input_pack.message, input_pack.endpoint) elif isinstance(input_pack.message, Message.Login): self._login_request(input_pack.message, input_pack.endpoint) elif isinstance(input_pack.message, Message.PlayerMovement): self._player_movement_request(input_pack.message, input_pack.endpoint) elif isinstance(input_pack.message, Message.PlayerCast): self._player_cast_request(input_pack.message, input_pack.endpoint) elif isinstance(input_pack.message, ServerSignal): if ServerSignal.NEW_ARENA_SIGNAL == input_pack.message: self._new_arena_signal() elif ServerSignal.COMPUTE_FRAME_SIGNAL == input_pack.message: self._compute_frame_signal() elif ServerSignal.ARENA_CREATED_SIGNAL == input_pack.message: self._arena_created_signal() else: logger.error( "Unknown message type: {} - Rejecting connection...". format(input_pack.message.__class__)) self._output_queue(OutputPack(None, input_pack.endpoint)) else: self._lost_connection(input_pack.endpoint)
async def main(db_config: dict, live=False, destroy=False) -> None: if destroy: logger.warning( "!!!!Running with destroy mode and this will destory the existing schema!!!! The script will wait for 5 secs to help you reevaluate the decision.") await asyncio.sleep(5) if live: logger.warning("Running in live mode. This will create tables and schemas") else: logger.info("Running in dry run mode. Just the DDLs will be printed") connection = None try: connection = await connect(db_config['conn_string']) async with connection.transaction(): if destroy: logger.warning("destroying the existing schema") await drop_schema(connection, db_config['schema'], live, destroy) await create_schema(connection, db_config['schema'], live) await create_tables(connection, db_config['table'], db_config['schema'], live) except DuplicateTableError as e: logger.error("Always run the fixtures on clean database. We don't support incremental migrations yet!", exc_info=e) finally: await connection.close()
async def _update(self, event: ChangeEvent): get_current_sql = "SELECT * FROM {} WHERE {} FOR UPDATE" lock_sql = "UPDATE {} set `pidal_c` = `pidal_c` | 1 WHERE {}" update_sql = "UPDATE {} set {} WHERE {}" client = await self.client.acquire() try: for i in event.values: if "before_values" not in i.keys( ) or "after_values" not in i.keys(): raise Exception( "update event value {} error lsn{}.".format( i, event.lsn.encode())) if not self._check_lock( event.table, i["before_values"]) or not self._check_lock( event.table, i["after_values"]): raise Exception("need lock_key {} error lsn{}.".format( self.tables[event.table], event.lsn.encode())) where = self._parser_where(self.tables[event.table], i["before_values"]) await client.begin() async with client.cursor() as cur: await cur.execute( get_current_sql.format(event.table, where)) current = await cur.fetchone() if current == i["after_values"]: # 数据已经修改过了。 await client.rollback() continue belong_zone_id = self._get_belong_zone_id( event.table, i["before_values"]) if belong_zone_id == self.zone_id: # 自己的修改 continue source_zone_id, _, data_version, _ = self._parser_pidal_c( i["after_values"]["pidal_c"]) _, _, current_version, is_lock = self._parser_pidal_c( current["pidal_c"]) if belong_zone_id != source_zone_id: logger.error( "error data modify in zone_id {} table {} data {}". format(event.source_zone_id, event.table, i["before_values"])) if belong_zone_id == self.zone_id and not is_lock: await cur.execute( lock_sql.format(event.table, where)) await client.commit() continue if current == i["before_values"]: await cur.execute( update_sql.format( event.table, self._parser_set(i["after_values"]), where)) await client.commit() continue if data_version < current_version: # 老数据 await client.rollback() elif data_version == current_version: # 数据校验不一致 if belong_zone_id == self.zone_id and not is_lock: await cur.execute( lock_sql.format(event.table, where)) await client.commit() else: await client.rollback() else: # 走到这里可能是数据落后版本太多,直接覆盖 await cur.execute( update_sql.format( event.table, self._parser_set(i["after_values"]), where)) await client.commit() finally: await client.rollback() self.client.release(client)
def parse_args(): """Parse arguments for module. Returns: argparse.Namespace: contains accessible arguments passed in to module """ parser = argparse.ArgumentParser() parser.add_argument("--training", help=("participants for training, space separated; " "e.g., W1 W2")) parser.add_argument("--validation", help=("participants for validation, space separated; " "e.g., P1 P2")) parser.add_argument("--testing", help=("participants for testing, space separated; " "e.g., P4 P5")) parser.add_argument("-f", "--data-path", help="path to h5 files for reading data") parser.add_argument("-o", "--output-path", help=("path to directory to save h5 files for " "training, validation, and testing")) parser.add_argument("-x", "--task-input", help=("input type; " "e.g., orientation, relativePosition, " "or jointAngle")) parser.add_argument("--input-label-request", help=("input label requests, space separated; " "e.g., all or Pelvis RightForearm")) parser.add_argument("-y", "--task-output", help="output type; e.g., orientation or jointAngle") parser.add_argument("--output-label-request", help=("output label requests, space separated; " "e.g., all or jRightElbow")) parser.add_argument("--aux-task-output", help=("auxiliary task output in addition " "to regular task output")) parser.add_argument("--aux-output-label-request", help="aux output label requests, space separated") args = parser.parse_args() if None in [args.training, args.validation, args.testing]: logger.info(("Participant numbers for training, validation, " "or testing dataset were not provided.")) parser.print_help() sys.exit() if None in [args.data_path, args.output_path]: logger.error("Data path or output path were not provided.") parser.print_help() sys.exit() if None in [args.task_input, args.input_label_request, args.task_output]: logger.error(("Task input and label requests " "or task output were not given.")) parser.print_help() sys.exit() if args.output_label_request is None: if args.task_input == args.task_output: logger.info("Will create h5 files with input data only.") else: logger.error("Label output requests were not given for the task.") parser.print_help() sys.exit() if args.aux_task_output == args.task_output: logger.error("Auxiliary task should not be the same as the main task.") parser.print_help() sys.exit() if (args.aux_task_output is not None and args.aux_output_label_request is None): logger.error("Need auxiliary output labels if using aux output task") parser.print_help() sys.exit() if args.task_input == args.task_output: if args.output_label_request is None: logger.info(("Will create h5 files with only input " "data for self-supervision tasks...")) else: logger.info("Will create h5 files with input and output data.") return args