def logs(client: ModelTrainingClient, train_id: str, file: str, follow: bool):
"""
\b
Stream training logs.
For this command, you must provide a training ID or path to file with one training.
The file must contain only one training.
\b
Usage example:
* odahuflowctl train delete --id examples-git
* odahuflowctl train delete -f train.yaml
\f
:param follow: Follow logs stream
:param client: Model training HTTP client
:param train_id: Model training ID
:param file: Path to the file with only one training
"""
check_id_or_file_params_present(train_id, file)
if file:
train = parse_resources_file_with_one_item(file).resource
if not isinstance(train, ModelTraining):
raise ValueError(
f'Model training expected, but {type(train)} provided')
train_id = train.id
for msg in client.log(train_id, follow):
print_logs(msg)
def logs(client: ModelPackagingClient, pack_id: str, file: str, follow: bool):
"""
\b
Stream packaging logs.
For this command, you must provide a packaging ID or path to file with one packaging.
The file must contain only one packaging.
The command will fail if you provide both arguments.
\b
Usage example:
* odahuflowctl pack delete --id examples-git
* odahuflowctl pack delete -f pack.yaml
\f
:param follow: Follow logs stream
:param client: Model packaging HTTP client
:param pack_id: Model packaging ID
:param file: Path to the file with only one packaging
"""
check_id_or_file_params_present(pack_id, file)
if file:
pack = parse_resources_file_with_one_item(file).resource
if not isinstance(pack, ModelPackaging):
raise ValueError(
f'Model packaging expected, but {type(pack)} provided')
pack_id = pack.id
for msg in client.log(pack_id, follow):
print_logs(msg)
def wait_training_finish(timeout: int, wait: bool, mt_id: str,
mt_client: ModelTrainingClient):
"""
Wait for training to finish according to command line arguments
:param wait:
:param timeout:
:param mt_id: Model Training name
:param mt_client: Model Training Client
"""
if not wait:
return
start = time.time()
if timeout <= 0:
raise Exception(
'Invalid --timeout argument: should be positive integer')
# We create a separate client for logs because it has the different timeout settings
log_mt_client = ModelTrainingClient.construct_from_other(mt_client)
log_mt_client.timeout = mt_client.timeout, LOG_READ_TIMEOUT_SECONDS
click.echo("Logs streaming...")
while True:
elapsed = time.time() - start
if elapsed > timeout:
raise Exception(TIMEOUT_ERROR_MESSAGE)
try:
mt = mt_client.get(mt_id)
if mt.status.state == TRAINING_SUCCESS_STATE:
click.echo(
f'Model {mt_id} was trained. Training took {round(time.time() - start)} seconds'
)
return
elif mt.status.state == TRAINING_FAILED_STATE:
raise Exception(f'Model training {mt_id} was failed.')
elif mt.status.state == "":
click.echo(
f"Can't determine the state of {mt.id}. Sleeping...")
else:
for msg in log_mt_client.log(mt.id, follow=True):
print_logs(msg)
except (WrongHttpStatusCode, HTTPException, RequestException,
APIConnectionException) as e:
LOGGER.info(
'Callback have not confirmed completion of the operation. Exception: %s',
str(e))
LOGGER.debug('Sleep before next request')
time.sleep(DEFAULT_WAIT_TIMEOUT)