async def create_table_if_not_exists(conn: Connection, schema: str,
table: str) -> None:
table_exists_bool = await table_exists(conn, schema, table)
if not table_exists_bool:
for table_name, ddl in table_ddl(table, schema).items():
logger.info(f"creating table {table_name}")
await conn.execute(ddl)
def _lost_connection(self, endpoint):
player = self._room.get_player_with_endpoint(endpoint)
if player:
player.set_endpoint(None)
logger.info("Player '{}' disconnected".format(
player.get_character()))
self._log_players()
def write_dataset(filepath_groups, variable, experiment_setup, requests):
"""Write data to training, validation, testing .h5 files.
Args:
filepath_groups (list): list of tuples associate files to data group
variable (str): the machine learning variable X or Y to be written to
experiment_setup (dict): map to reference task for variable
requests (dict): requests to read from files to store with variable
"""
for filepaths, group in filepath_groups:
logger.info(f"Writing {variable} to the {group} set...")
h5_filename = args.output_path + "/" + group + ".h5"
with h5py.File(h5_filename, "a") as h5_file:
dataset = read_h5(filepaths, requests)
for filename in dataset.keys():
temp_dataset = None
for j, data_group in enumerate(experiment_setup[variable]):
if temp_dataset is None:
temp_dataset = dataset[filename][data_group]
else:
temp_dataset = np.append(temp_dataset,
dataset[filename][data_group],
axis=1)
try:
h5_file.create_dataset(filename + "/" + variable,
data=temp_dataset)
except KeyError:
logger.info(f"{filename} does not contain {data_group}")
async def create_tables(connection: Connection, table_name: str, schema: str, live=False) -> None:
for table_name, ddl in table_ddl(table_name, schema).items():
logger.info(f"creating table {table_name}")
if live:
await connection.execute(ddl)
else:
logger.info(ddl)
async def create_schema(connection: Connection, schema: str, live=False) -> None:
schema_create = f"CREATE SCHEMA IF NOT EXISTS {schema}"
logger.info(f"creating schema {schema}")
if live:
await connection.execute(schema_create)
else:
logger.info(schema_create)
async def drop_schema(connection: Connection, schema: str, live=False, destroy=False) -> None:
schema_drop = f"DROP SCHEMA IF EXISTS {schema} cascade"
logger.warning(f"dropping schema {schema}")
if live and destroy:
await connection.execute(schema_drop)
else:
logger.info(schema_drop)
def _register_player(self, character, endpoint):
if 1 > len(character) or -1 == string.ascii_uppercase.find(character):
logger.warning(
"Login attempt with invalid character: {}".format(character))
return Message.LoginStatus.INVALID_CHARACTER
status = self._room.add_player(character, endpoint)
if Room.ADDITION_SUCCESSFUL == status:
logger.info(
"Player '{}' registered successfully".format(character))
return Message.LoginStatus.LOGGED
elif Room.ADDITION_REUSE == status:
logger.info("Player '{}' reconnected".format(character))
return Message.LoginStatus.RECONNECTION
elif Room.ADDITION_ERR_COMPLETE == status:
logger.debug("Player '{}' tried to register: room complete".format(
character))
return Message.LoginStatus.ROOM_COMPLETED
elif Room.ADDITION_ERR_ALREADY_EXISTS == status:
logger.debug(
"Player '{}' tried to register: already exists".format(
character))
return Message.LoginStatus.ALREADY_EXISTS
async def insert_many(conn: Connection, schema: str, table: str, columns: list,
values: Iterable[list]) -> None:
num_columns = len(columns)
place_holders = ",".join(
list(map(lambda x: f'${x}', range(1, num_columns + 1))))
columns_str = ",".join(columns)
insert_query = f"INSERT INTO {schema}.{table} ({columns_str}) VALUES ({place_holders}) ON CONFLICT DO NOTHING"
logger.info(f"executing query {insert_query}")
logger.info(values)
await conn.executemany(insert_query, values)
def post(self, request) -> HttpResponse:
"""
Handle post request to write chunks from incoming ResumableJS requests
If one of the requests notices all the chunks have been fully written
it will write the full file to
<self.FILE_UPLOAD_DIR>/<user.plant.name_key>/<uploaded_file_name>
"""
total_chunks = int(request.POST.get('resumableTotalChunks'))
chunk_num = int(request.POST.get('resumableChunkNumber', 1))
uploaded_file_name = request.POST.get('resumableFilename')
resumable_identifier = request.POST.get('resumableIdentifier', 'error')
chunk_data = request.FILES['file']
user = request.user
plant_name_key = user.plant.name_key
# Get - tmp file dir path for the ResumableJS upload chunks
# - target file path
target_file_path = self.target_file_path(plant_name_key,
uploaded_file_name)
chunk_dir = self.FILE_UPLOAD_DIR / plant_name_key / resumable_identifier
# Save this chunk data using a lock file
self._save_chunk_data(uploaded_file_name, chunk_num, chunk_data,
chunk_dir)
# Check if the all chunk files created in all Resumable JS Requests
all_chunk_paths = self._all_chunk_paths(uploaded_file_name, chunk_dir,
total_chunks)
all_chunks_exists = all([p.exists() for p in all_chunk_paths])
if all_chunks_exists:
# Make sure all files are finished writing, but do not wait forever
tried = 0
while self._all_chunks_not_written(chunk_dir, total_chunks):
tried += 1
if tried >= 5:
error_msg = f'Error uploading files with temp_dir: {chunk_dir!r}'
logger.error(f'[{self.LOG_PREFIX}] {error_msg}')
return HttpResponseServerError(error_msg)
time.sleep(1)
# If all chunks writen create full file and remove chunk dir/files
self._create_full_file_from_chunks(target_file_path,
all_chunk_paths, chunk_dir)
logger.info(
f'[{self.LOG_PREFIX}] User {user!r} successfully created '
f'file {target_file_path}')
return HttpResponse(status=200)
def connect(self, ip, port):
try:
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect((ip, port))
connection.setblocking(False)
self._selector.register(connection, selectors.EVENT_READ,
self.Operation.READ)
logger.info("New connection to {}:{}".format(ip, port))
return connection
except OSError as error:
logger.critical("Can not connect to {}:{}, error: {}".format(
ip, port, error.errno))
return None
def __init__(self, players, points, arena_size, seed):
PackageQueue.__init__(self)
self._active = True
self._room = Room(players, points)
self._arena_size = arena_size
self._seed = seed
self._arena = None
self._arena_enabled = False
self._last_frame_time_stamp = 0
self._last_waiting_time = 0
logger.info("Required players: {} - Points to win: {}".format(
players, points))
def solve():
longest = (0, [])
for a in range(1, 10):
for b in range(a + 1, 10):
for c in range(b + 1, 10):
for d in range(c + 1, 10):
digits = [a, b, c, d]
length = consecutive_positive_integers(targets(digits))
logger.info(f"{digits}: {length}")
if length > longest[0]:
longest = (length, digits)
return join_digits(longest[1])
def solve(limit=13):
odd_periods = 0
for n in range(2, limit + 1):
a0, *period = period_of_root(n)
if len(period) % 2 != 0:
odd_periods += 1
if period:
logger.info(
f"√{n} = [{a0}; {tuple(period)}], period={len(period)}")
else:
logger.debug(f"√{n} = {sqrt(n)}")
return odd_periods
def listen(self, port):
try:
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.setblocking(False)
server_socket.bind(("0.0.0.0", port))
server_socket.listen()
self._selector.register(server_socket, selectors.EVENT_READ,
self.Operation.ACCEPT)
logger.info("Listening on port: {}".format(port))
return server_socket
except OSError as error:
logger.critical(
"Problem initializing the server on port {}, error: {}".format(
port, error.errno))
if (98 == error.errno):
logger.critical("Port {} is already in use".format(port))
return None
def new_arena(self):
pre_time_stamp = time.time()
seed = self._seed if "" != self._seed else ServerManager.compute_random_seed(
RANDOM_SEED_SIZE)
logger.info("Load arena - size: {}, seed: {}".format(
self._arena_size, seed))
self._arena = Arena(self._arena_size, seed)
position_list = self._arena.compute_player_origins(
self._room.get_size())
for i, player in enumerate(self._room.get_player_list()):
control = self._arena.create_player(player.get_character(),
position_list[i])
player.set_control(control)
post_time_stamp = time.time()
logger.info("Load arena - done! {0:.2}s".format(post_time_stamp -
pre_time_stamp))
self._server_signal(ServerSignal.ARENA_CREATED_SIGNAL, 0)
async def main(db_config: dict, kafka_config: dict, buffer_limit: int = 100):
pool = await init_pool(db_config['conn_string'])
consumer = await kafka_consumer(kafka_config)
async with pool.acquire() as connection:
await create_table_if_not_exists(connection, db_config['schema'],
db_config['table'])
try:
buffer = []
async for message in consumer:
logger.info(f"received message : {message}")
value = message.value
value['kafka_partition_offset_id'] = kafka_meta_id(message)
buffer.append(value)
if len(buffer) >= buffer_limit:
logger.info(
"Buffer is full. Going to attempt insert into table")
await insert_into_db(connection, db_config['schema'],
db_config['table'], buffer)
await consumer.commit()
buffer.clear()
finally:
await consumer.stop()
await pool.close()
async def main(db_config: dict, live=False, destroy=False) -> None:
if destroy:
logger.warning(
"!!!!Running with destroy mode and this will destory the existing schema!!!! The script will wait for 5 secs to help you reevaluate the decision.")
await asyncio.sleep(5)
if live:
logger.warning("Running in live mode. This will create tables and schemas")
else:
logger.info("Running in dry run mode. Just the DDLs will be printed")
connection = None
try:
connection = await connect(db_config['conn_string'])
async with connection.transaction():
if destroy:
logger.warning("destroying the existing schema")
await drop_schema(connection, db_config['schema'], live, destroy)
await create_schema(connection, db_config['schema'], live)
await create_tables(connection, db_config['table'], db_config['schema'], live)
except DuplicateTableError as e:
logger.error("Always run the fixtures on clean database. We don't support incremental migrations yet!",
exc_info=e)
finally:
await connection.close()
def _create_full_file_from_chunks(self, target_file_path: Path,
all_chunk_paths: List[Path],
chunk_dir: Path):
"""
Once all the chunk data has been written create the full file
with the aggregated chunks
"""
# Make sure some other chunk didn't trigger file reconstruction
if target_file_path.exists():
logger.debug(
f'[{self.LOG_PREFIX}] File {target_file_path!r} exists already. Overwriting..'
)
target_file_path.unlink()
# Save file from all uploaded chunk data
with open(target_file_path, "ab") as fp:
for p in all_chunk_paths:
with open(p, 'rb') as stored_chunk_file:
fp.write(stored_chunk_file.read())
logger.info(
f'[{self.LOG_PREFIX}] File saved to {target_file_path!r}')
# Remove the chunk dir all all files in it
shutil.rmtree(chunk_dir)
async def execute(self, client: HttpClientLike) -> CheckResult:
results = dict()
logger.info(f"GET url: {self.url}")
response = await client.get(self.url, retries=3)
if self.regex_checks is not None:
logger.info(f"executing extra regex checks for url: {self.url}")
for name, check_func in self.regex_checks.items():
logger.info(
f"executing regex check : {name} for url: {self.url}")
results[name] = check_func(response.body)
return CheckResult(self.url, response.code, response.time, results,
response.error)
async def schedule(client: HttpClientLike, kafka_config: dict,
kafka_p: AIOKafkaProducer, check: Check, repeat_in_s: int):
logger.info(
f"scheduling observer for {check.url} every {repeat_in_s} seconds")
while True:
result = await check.execute(client)
logger.info(
f"Results for url: {check.url} - results:{result.to_json()}")
await kafka_send(kafka_p, kafka_config['topic_name'],
bytes(result.to_json(), 'utf-8'))
await asyncio.sleep(repeat_in_s)
logger.info(
f"repeating check for {check.url} after {repeat_in_s} seconds")
def fit(models, optims, epochs, dataloaders, training_criterion,
validation_criteria, schedulers, device, model_file_path,
teacher_forcing_ratio=0.0, use_attention=False,
norm_quaternions=False, schedule_rate=1.0):
"""Fit a seq2seq model to data, logging training and validation loss.
Args:
models (tuple): tuple containing the encoder and decoder
optims (tuple): tuple containing the encoder and decoder optimizers for
training
epochs (int): number of epochs to train for
dataloaders (tuple): tuple containing the training dataloader and val
dataloader.
training_criterion (nn.Module): criterion for backpropagation during
training
validation_criteria (list): list of criteria for validation
schedulers (list): list of schedulers to control learning rate for
optimizers
device (torch.device): device to place data on
model_file_path (str): where to save the model when validation loss
reaches new minimum
teacher_forcing_ratio (float, optional): percent of the time to use
teacher forcing in the decoder. Defaults to 0.0.
use_attention (bool, optional): whether decoder uses attention or not.
Defaults to False.
norm_quaternions (bool, optional): whether quaternions should be
normalized after they are output from the decoder.
Defaults to False.
schedule_rate (float, optional): rate to increase or decrease teacher
forcing ratio. Defaults to 1.0.
"""
train_dataloader, val_dataloader = dataloaders
min_val_loss = math.inf
for epoch in range(epochs):
losses = []
total_time = 0
logger.info(f"Epoch {epoch+1} / {epochs}")
for index, data in enumerate(train_dataloader, 0):
with Timer() as timer:
loss = loss_batch(data, models,
optims, training_criterion, device,
use_attention=use_attention,
norm_quaternions=norm_quaternions)
losses.append(loss)
total_time += timer.interval
if index % (len(train_dataloader) // 10) == 0:
logger.info((f"Total time elapsed: {total_time} - "
f"Batch Number: {index} / {len(train_dataloader)}"
f" - Training loss: {loss}"
))
val_loss = []
for validation_criterion in validation_criteria:
with torch.no_grad():
val_losses = [loss_batch(data, models,
None, validation_criterion, device,
use_attention=use_attention,
norm_quaternions=norm_quaternions)
for _, data in enumerate(val_dataloader, 0)]
val_loss.append(np.sum(val_losses) / len(val_losses))
loss = np.sum(losses) / len(losses)
for scheduler in schedulers:
scheduler.step()
val_loss_strs = ", ".join(map(str, val_loss))
logger.info(f"Training Loss: {loss} - Val Loss: {val_loss_strs}")
teacher_forcing_ratio *= schedule_rate
if val_loss[0] < min_val_loss:
min_val_loss = val_loss[0]
logger.info(f"Saving model to {model_file_path}")
torch.save({
"encoder_state_dict": models[0].state_dict(),
"decoder_state_dict": models[1].state_dict(),
"optimizerA_state_dict": optims[0].state_dict(),
"optimizerB_state_dict": optims[1].state_dict(),
}, model_file_path)
parser.add_argument("--num-layers",
help="number of layers in Transformer Encoder")
args = parser.parse_args()
if args.data_path_parent is None:
parser.print_help()
return args
if __name__ == "__main__":
args = parse_args()
for arg in vars(args):
logger.info(f"{arg} - {getattr(args, arg)}")
logger.info("Starting Transformer testing...")
logger.info(f"Device count: {str(torch.cuda.device_count())}")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
logger.info("Testing on {}...".format(device))
seq_length = int(args.seq_length) // int(args.downsample)
data_paths = [
args.data_path_parent + "/" + name
for name in os.listdir(args.data_path_parent)
if os.path.isdir(args.data_path_parent + "/" + name)
]
from common.logging import logger
limit = 100
count = 0
for b in range(1, limit):
for e in range(1, limit):
n = b ** e
if len(str(n)) == e:
count += 1
logger.info(f"{count}: {b}^{e} = {n}")
print(count)
parser.add_argument("--dropout", help="dropout percentage in Transformer")
parser.add_argument("--num-layers", help="number of layers in Transformer")
args = parser.parse_args()
if args.data_path is None:
parser.print_help()
return args
if __name__ == "__main__":
args = parse_args()
for arg in vars(args):
logger.info(f"{arg} - {getattr(args, arg)}")
logger.info("Starting Transformer training...")
logger.info(f"Device count: {torch.cuda.device_count()}")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
logger.info(f"Training on {device}...")
seq_length = int(args.seq_length) // int(args.downsample)
assert seq_length % int(args.in_out_ratio) == 0
lr = float(args.learning_rate)
normalize = True
train_dataloader, norm_data = load_dataloader(args, "training", normalize)
default=False,
action="store_true")
parser.add_argument("--attention",
help="use decoder with specified attention",
default="general")
args = parser.parse_args()
return args
if __name__ == "__main__":
args = parse_args()
for arg in vars(args):
logger.info("{} - {}".format(arg, getattr(args, arg)))
logger.info("Starting seq2seq model testing...")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
seq_length = int(args.seq_length)
stride = int(args.stride)
batch_size = int(args.batch_size)
data_paths = [
args.data_path_parent + "/" + name
for name in os.listdir(args.data_path_parent)
if os.path.isdir(args.data_path_parent + "/" + name)
]
def _log_players(self):
connected_players = self._room.get_character_list_with_endpoints()
logger.info("Logged players: {} - Connected players: {}".format(
self._room.get_character_list(), connected_players))
def __init__(self, players, points, arena_size, seed):
logger.info("Server version: {}".format(version.CURRENT))
self._server_manager = ServerManager(players, points, arena_size, seed)
def fit(model,
optimizer,
scheduler,
epochs,
dataloaders,
training_criterion,
validation_criteria,
device,
model_file_path,
full_transformer=False,
min_val_loss=math.inf):
"""Fit a Transformer model to data, logging training and validation loss.
Args:
model (nn.Module): the model to train
optimizer (tuple): the optimizer for training
scheduler (list): scheduler to control learning rate for
optimizers
epochs (int): number of epochs to train for
dataloaders (tuple): tuple containing the training dataloader and val
dataloader.
training_criterion (nn.Module): criterion for backpropagation during
training
validation_criteria (list): list of criteria for validation
device (torch.device): device to place data on
model_file_path (str): where to save the model when validation loss
reaches new minimum
full_transformer (bool): whether the model is a full transformer and
needs to run inference for evaluation
min_val_loss (float, optional): minimum validation loss
Returns:
float: minimum validation loss reached during training
"""
train_dataloader, val_dataloader = dataloaders
total_time = 0
for epoch in range(epochs):
losses = 0
logger.info("Epoch {}".format(epoch + 1))
avg_loss = 0
for index, data in enumerate(train_dataloader, 0):
with Timer() as timer:
loss = loss_batch(model,
optimizer,
data,
training_criterion,
device,
full_transformer=full_transformer)
losses += loss
avg_loss += loss
total_time += timer.interval
if index % (len(train_dataloader) // 10) == 0 and index != 0:
avg_training_loss = avg_loss / (len(train_dataloader) // 10)
logger.info((f"Total time elapsed: {total_time}"
" - "
f"Batch number: {index} / {len(train_dataloader)}"
" - "
f"Training loss: {avg_training_loss}"
" - "
f"LR: {optimizer.param_groups[0]['lr']}"))
avg_loss = 0
val_loss = []
for validation_criterion in validation_criteria:
with torch.no_grad():
val_losses = [
loss_batch(model,
None,
data,
validation_criterion,
device,
full_transformer=full_transformer)
for _, data in enumerate(val_dataloader, 0)
]
val_loss.append(np.sum(val_losses) / len(val_losses))
loss = losses / len(train_dataloader)
scheduler.step()
val_loss_str = ", ".join(map(str, val_loss))
logger.info(f"Epoch {epoch+1} - "
f"Training Loss: {loss} - "
f"Val Loss: {val_loss_str}")
if full_transformer:
inference_loss = []
for validation_criterion in validation_criteria:
with torch.no_grad():
inference_losses = [
inference(model, data, validation_criterion, device)
for _, data in enumerate(val_dataloader, 0)
]
inference_loss.append(
np.sum(inference_losses) / len(inference_losses))
inference_loss_str = ", ".join(map(str, inference_loss))
logger.info(f"Inference Loss: {inference_loss_str}")
if val_loss[0] < min_val_loss:
min_val_loss = val_loss[0]
logger.info(f"Saving model to {model_file_path}")
torch.save(
{
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
}, model_file_path)
return min_val_loss
help="will use decoder with given attention method",
default="general")
args = parser.parse_args()
if args.data_path is None:
parser.print_help()
return args
if __name__ == "__main__":
args = parse_args()
for arg in vars(args):
logger.info(f"{arg} - {getattr(args, arg)}")
logger.info("Starting seq2seq model training...")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
seq_length = int(args.seq_length)
stride = int(args.stride)
lr = float(args.learning_rate)
assert seq_length % int(args.in_out_ratio) == 0
normalize = True
train_dataloader, norm_data = load_dataloader(args, "training",
normalize, norm_data=None)
val_dataloader, _ = load_dataloader(args, "validation",
def parse_args():
"""Parse arguments for module.
Returns:
argparse.Namespace: contains accessible arguments passed in to module
"""
parser = argparse.ArgumentParser()
parser.add_argument("--training",
help=("participants for training, space separated; "
"e.g., W1 W2"))
parser.add_argument("--validation",
help=("participants for validation, space separated; "
"e.g., P1 P2"))
parser.add_argument("--testing",
help=("participants for testing, space separated; "
"e.g., P4 P5"))
parser.add_argument("-f",
"--data-path",
help="path to h5 files for reading data")
parser.add_argument("-o",
"--output-path",
help=("path to directory to save h5 files for "
"training, validation, and testing"))
parser.add_argument("-x",
"--task-input",
help=("input type; "
"e.g., orientation, relativePosition, "
"or jointAngle"))
parser.add_argument("--input-label-request",
help=("input label requests, space separated; "
"e.g., all or Pelvis RightForearm"))
parser.add_argument("-y",
"--task-output",
help="output type; e.g., orientation or jointAngle")
parser.add_argument("--output-label-request",
help=("output label requests, space separated; "
"e.g., all or jRightElbow"))
parser.add_argument("--aux-task-output",
help=("auxiliary task output in addition "
"to regular task output"))
parser.add_argument("--aux-output-label-request",
help="aux output label requests, space separated")
args = parser.parse_args()
if None in [args.training, args.validation, args.testing]:
logger.info(("Participant numbers for training, validation, "
"or testing dataset were not provided."))
parser.print_help()
sys.exit()
if None in [args.data_path, args.output_path]:
logger.error("Data path or output path were not provided.")
parser.print_help()
sys.exit()
if None in [args.task_input, args.input_label_request, args.task_output]:
logger.error(("Task input and label requests "
"or task output were not given."))
parser.print_help()
sys.exit()
if args.output_label_request is None:
if args.task_input == args.task_output:
logger.info("Will create h5 files with input data only.")
else:
logger.error("Label output requests were not given for the task.")
parser.print_help()
sys.exit()
if args.aux_task_output == args.task_output:
logger.error("Auxiliary task should not be the same as the main task.")
parser.print_help()
sys.exit()
if (args.aux_task_output is not None
and args.aux_output_label_request is None):
logger.error("Need auxiliary output labels if using aux output task")
parser.print_help()
sys.exit()
if args.task_input == args.task_output:
if args.output_label_request is None:
logger.info(("Will create h5 files with only input "
"data for self-supervision tasks..."))
else:
logger.info("Will create h5 files with input and output data.")
return args
