async def _delete(self, event: ChangeEvent):
get_current_sql = "SELECT * FROM {} WHERE {} FOR UPDATE"
delete_sql = "DELETE FROM {} WHERE {}"
client = await self.client.acquire()
try:
for i in event.values:
if "values" not in i.keys():
raise Exception(
"update event value {} error lsn{}.".format(
i, event.lsn.encode()))
if not self._check_lock(event.table, i["values"]):
raise Exception("need lock_key {} error lsn{}.".format(
self.tables[event.table], event.lsn.encode()))
where = self._parser_where(self.tables[event.table],
i["values"])
await client.begin()
async with client.cursor() as cur:
await cur.execute(
get_current_sql.format(event.table, where))
current = await cur.fetchone()
if not current: # 数据已经删除
await client.rollback()
continue
await cur.execute(delete_sql.format(event.table, where))
await client.commit()
if current != i["values"]: # 需要删除数据
# 走到这里说明有数据，切和 event 的不一致。
logger.error(
"error data insert in zone_id {} table {} data {}".
format(self.zone_id, event.table, current))
finally:
await client.rollback()
self.client.release(client)
async def _insert(self, event: ChangeEvent):
get_current_sql = "SELECT * FROM {} WHERE {} FOR UPDATE"
insert_sql = "INSERT INTO {} ({}) VALUES ({})"
client = await self.client.acquire()
try:
for i in event.values:
if "values" not in i.keys():
raise Exception(
"update event value {} error lsn{}.".format(
i, event.lsn.encode()))
if not self._check_lock(event.table, i["values"]):
raise Exception("need lock_key {} error lsn{}.".format(
self.tables[event.table], event.lsn.encode()))
where = self._parser_where(self.tables[event.table],
i["values"])
await client.begin()
async with client.cursor() as cur:
await cur.execute(
get_current_sql.format(event.table, where))
current = await cur.fetchone()
if current == i["values"]: # 数据已经插入。
await client.rollback()
continue
if not current:
# 正常插入数据
cl = ", ".join(
["`{}`".format(c) for c in i["values"].keys()])
va = ", ".join(
["'{}'".format(c) for c in i["values"].values()])
await cur.execute(
insert_sql.format(event.table, cl, va))
await client.commit()
# 走到这里说明有数据，切和 event 的不一致。
belong_zone_id = self._get_belong_zone_id(
event.table, i["values"])
if belong_zone_id == self.zone_id:
# 自己的修改
await client.rollback()
continue
source_zone_id, _, data_version, _ = self._parser_pidal_c(
i["values"]["pidal_c"])
if belong_zone_id != source_zone_id:
logger.error(
"error data insert in zone_id {} table {} data {}"
.format(event.source_zone_id, event.table,
i["before_values"]))
await client.rollback()
continue
# 走到这里说明来源的数据是对的，但是当前的数据不知道什么原因保留现场，不自改。
logger.error(
"error data insert in zone_id {} table {} data {}".
format(self.zone_id, event.table, current))
await client.rollback()
finally:
await client.rollback()
self.client.release(client)
def _player_cast_request(self, player_cast_message, endpoint):
player = self._check_player_for_event(endpoint)
if False: #Check that the skill exists
logger.error("Unexpected skill from player '{}'".format(
player.get_character()))
self._output_queue.put(OutputPack("", endpoint))
return
control = player.get_control()
if control:
control.cast(player_cast_message.skill_id)
def _player_movement_request(self, player_movement_message, endpoint):
player = self._check_player_for_event(endpoint)
if not Direction.is_orthogonal(player_movement_message.direction):
logger.error("Unexpected movement value from player '{}'".format(
player.get_character()))
self._output_queue.put(OutputPack("", endpoint))
return
control = player.get_control()
if control:
control.move(player_movement_message.direction)
def post(self, request) -> HttpResponse:
"""
Handle post request to write chunks from incoming ResumableJS requests
If one of the requests notices all the chunks have been fully written
it will write the full file to
<self.FILE_UPLOAD_DIR>/<user.plant.name_key>/<uploaded_file_name>
"""
total_chunks = int(request.POST.get('resumableTotalChunks'))
chunk_num = int(request.POST.get('resumableChunkNumber', 1))
uploaded_file_name = request.POST.get('resumableFilename')
resumable_identifier = request.POST.get('resumableIdentifier', 'error')
chunk_data = request.FILES['file']
user = request.user
plant_name_key = user.plant.name_key
# Get - tmp file dir path for the ResumableJS upload chunks
# - target file path
target_file_path = self.target_file_path(plant_name_key,
uploaded_file_name)
chunk_dir = self.FILE_UPLOAD_DIR / plant_name_key / resumable_identifier
# Save this chunk data using a lock file
self._save_chunk_data(uploaded_file_name, chunk_num, chunk_data,
chunk_dir)
# Check if the all chunk files created in all Resumable JS Requests
all_chunk_paths = self._all_chunk_paths(uploaded_file_name, chunk_dir,
total_chunks)
all_chunks_exists = all([p.exists() for p in all_chunk_paths])
if all_chunks_exists:
# Make sure all files are finished writing, but do not wait forever
tried = 0
while self._all_chunks_not_written(chunk_dir, total_chunks):
tried += 1
if tried >= 5:
error_msg = f'Error uploading files with temp_dir: {chunk_dir!r}'
logger.error(f'[{self.LOG_PREFIX}] {error_msg}')
return HttpResponseServerError(error_msg)
time.sleep(1)
# If all chunks writen create full file and remove chunk dir/files
self._create_full_file_from_chunks(target_file_path,
all_chunk_paths, chunk_dir)
logger.info(
f'[{self.LOG_PREFIX}] User {user!r} successfully created '
f'file {target_file_path}')
return HttpResponse(status=200)
async def send(p: AIOKafkaProducer,
topic: str,
data: bytes,
retried=False) -> None:
try:
future = await p.send(topic, data)
await future
except KafkaTimeoutError as e:
logger.error("kafka producer send failed", exc_info=e)
# In case of timeout, we retry once
if not retried:
await send(p, topic, data, retried=True)
else:
raise e
def _check_player_for_event(self, endpoint):
player = self._room.get_player_with_endpoint(endpoint)
if not player:
logger.error("Received event from unknown player")
self._output_queue.put(OutputPack("", endpoint))
return
if not self._arena_enabled:
logger.error(
"Received player event from '{}' without available arena".
format(player.get_character()))
self._output_queue.put(OutputPack("", endpoint))
return
return player
async def get(self, url, retries) -> ClientResponse:
try:
start = time.perf_counter()
response = await self.session.get(url)
total_time = time.perf_counter() - start
text = await response.text()
if response.status < 400:
return ClientResponse(response.status, total_time, text, None)
else:
return ClientResponse(response.status, total_time, None, response.reason)
except ClientConnectionError as e:
if retries > 0:
logger.error(f"Request failed while GET on url: {url}. Retrying")
await self.get(self.session, retries - 1)
else:
raise e
def process_requests(self):
while self._active:
input_pack = self._input_queue.get()
if input_pack.message:
if isinstance(input_pack.message, Message.Version):
self._info_server_request(input_pack.message,
input_pack.endpoint)
elif isinstance(input_pack.message, Message.Login):
self._login_request(input_pack.message,
input_pack.endpoint)
elif isinstance(input_pack.message, Message.PlayerMovement):
self._player_movement_request(input_pack.message,
input_pack.endpoint)
elif isinstance(input_pack.message, Message.PlayerCast):
self._player_cast_request(input_pack.message,
input_pack.endpoint)
elif isinstance(input_pack.message, ServerSignal):
if ServerSignal.NEW_ARENA_SIGNAL == input_pack.message:
self._new_arena_signal()
elif ServerSignal.COMPUTE_FRAME_SIGNAL == input_pack.message:
self._compute_frame_signal()
elif ServerSignal.ARENA_CREATED_SIGNAL == input_pack.message:
self._arena_created_signal()
else:
logger.error(
"Unknown message type: {} - Rejecting connection...".
format(input_pack.message.__class__))
self._output_queue(OutputPack(None, input_pack.endpoint))
else:
self._lost_connection(input_pack.endpoint)
async def main(db_config: dict, live=False, destroy=False) -> None:
if destroy:
logger.warning(
"!!!!Running with destroy mode and this will destory the existing schema!!!! The script will wait for 5 secs to help you reevaluate the decision.")
await asyncio.sleep(5)
if live:
logger.warning("Running in live mode. This will create tables and schemas")
else:
logger.info("Running in dry run mode. Just the DDLs will be printed")
connection = None
try:
connection = await connect(db_config['conn_string'])
async with connection.transaction():
if destroy:
logger.warning("destroying the existing schema")
await drop_schema(connection, db_config['schema'], live, destroy)
await create_schema(connection, db_config['schema'], live)
await create_tables(connection, db_config['table'], db_config['schema'], live)
except DuplicateTableError as e:
logger.error("Always run the fixtures on clean database. We don't support incremental migrations yet!",
exc_info=e)
finally:
await connection.close()
async def _update(self, event: ChangeEvent):
get_current_sql = "SELECT * FROM {} WHERE {} FOR UPDATE"
lock_sql = "UPDATE {} set `pidal_c` = `pidal_c` | 1 WHERE {}"
update_sql = "UPDATE {} set {} WHERE {}"
client = await self.client.acquire()
try:
for i in event.values:
if "before_values" not in i.keys(
) or "after_values" not in i.keys():
raise Exception(
"update event value {} error lsn{}.".format(
i, event.lsn.encode()))
if not self._check_lock(
event.table,
i["before_values"]) or not self._check_lock(
event.table, i["after_values"]):
raise Exception("need lock_key {} error lsn{}.".format(
self.tables[event.table], event.lsn.encode()))
where = self._parser_where(self.tables[event.table],
i["before_values"])
await client.begin()
async with client.cursor() as cur:
await cur.execute(
get_current_sql.format(event.table, where))
current = await cur.fetchone()
if current == i["after_values"]: # 数据已经修改过了。
await client.rollback()
continue
belong_zone_id = self._get_belong_zone_id(
event.table, i["before_values"])
if belong_zone_id == self.zone_id:
# 自己的修改
continue
source_zone_id, _, data_version, _ = self._parser_pidal_c(
i["after_values"]["pidal_c"])
_, _, current_version, is_lock = self._parser_pidal_c(
current["pidal_c"])
if belong_zone_id != source_zone_id:
logger.error(
"error data modify in zone_id {} table {} data {}".
format(event.source_zone_id, event.table,
i["before_values"]))
if belong_zone_id == self.zone_id and not is_lock:
await cur.execute(
lock_sql.format(event.table, where))
await client.commit()
continue
if current == i["before_values"]:
await cur.execute(
update_sql.format(
event.table,
self._parser_set(i["after_values"]), where))
await client.commit()
continue
if data_version < current_version:
# 老数据
await client.rollback()
elif data_version == current_version:
# 数据校验不一致
if belong_zone_id == self.zone_id and not is_lock:
await cur.execute(
lock_sql.format(event.table, where))
await client.commit()
else:
await client.rollback()
else:
# 走到这里可能是数据落后版本太多,直接覆盖
await cur.execute(
update_sql.format(
event.table,
self._parser_set(i["after_values"]), where))
await client.commit()
finally:
await client.rollback()
self.client.release(client)
def parse_args():
"""Parse arguments for module.
Returns:
argparse.Namespace: contains accessible arguments passed in to module
"""
parser = argparse.ArgumentParser()
parser.add_argument("--training",
help=("participants for training, space separated; "
"e.g., W1 W2"))
parser.add_argument("--validation",
help=("participants for validation, space separated; "
"e.g., P1 P2"))
parser.add_argument("--testing",
help=("participants for testing, space separated; "
"e.g., P4 P5"))
parser.add_argument("-f",
"--data-path",
help="path to h5 files for reading data")
parser.add_argument("-o",
"--output-path",
help=("path to directory to save h5 files for "
"training, validation, and testing"))
parser.add_argument("-x",
"--task-input",
help=("input type; "
"e.g., orientation, relativePosition, "
"or jointAngle"))
parser.add_argument("--input-label-request",
help=("input label requests, space separated; "
"e.g., all or Pelvis RightForearm"))
parser.add_argument("-y",
"--task-output",
help="output type; e.g., orientation or jointAngle")
parser.add_argument("--output-label-request",
help=("output label requests, space separated; "
"e.g., all or jRightElbow"))
parser.add_argument("--aux-task-output",
help=("auxiliary task output in addition "
"to regular task output"))
parser.add_argument("--aux-output-label-request",
help="aux output label requests, space separated")
args = parser.parse_args()
if None in [args.training, args.validation, args.testing]:
logger.info(("Participant numbers for training, validation, "
"or testing dataset were not provided."))
parser.print_help()
sys.exit()
if None in [args.data_path, args.output_path]:
logger.error("Data path or output path were not provided.")
parser.print_help()
sys.exit()
if None in [args.task_input, args.input_label_request, args.task_output]:
logger.error(("Task input and label requests "
"or task output were not given."))
parser.print_help()
sys.exit()
if args.output_label_request is None:
if args.task_input == args.task_output:
logger.info("Will create h5 files with input data only.")
else:
logger.error("Label output requests were not given for the task.")
parser.print_help()
sys.exit()
if args.aux_task_output == args.task_output:
logger.error("Auxiliary task should not be the same as the main task.")
parser.print_help()
sys.exit()
if (args.aux_task_output is not None
and args.aux_output_label_request is None):
logger.error("Need auxiliary output labels if using aux output task")
parser.print_help()
sys.exit()
if args.task_input == args.task_output:
if args.output_label_request is None:
logger.info(("Will create h5 files with only input "
"data for self-supervision tasks..."))
else:
logger.info("Will create h5 files with input and output data.")
return args
