def build_client(resource: OdahuflowCloudResourceUpdatePair,
api_client: RemoteAPIClient) -> typing.Optional[object]:
"""
Build client for particular resource (e.g. it builds ModelTrainingClient for ModelTraining resource)
:param resource: target resource
:param api_client: base API client to extract connection options from
:return: remote client or None
"""
if isinstance(resource.resource, ModelTraining):
return ModelTrainingClient.construct_from_other(api_client)
elif isinstance(resource.resource, ModelDeployment):
return ModelDeploymentClient.construct_from_other(api_client)
elif isinstance(resource.resource, Connection):
return ConnectionClient.construct_from_other(api_client)
elif isinstance(resource.resource, ToolchainIntegration):
return ToolchainIntegrationClient.construct_from_other(api_client)
elif isinstance(resource.resource, ModelRoute):
return ModelRouteClient.construct_from_other(api_client)
elif isinstance(resource.resource, ModelPackaging):
return ModelPackagingClient.construct_from_other(api_client)
elif isinstance(resource.resource, PackagingIntegration):
return PackagingIntegrationClient.construct_from_other(api_client)
else:
raise InvalidResourceType('{!r} is invalid resource '.format(
resource.resource))
def wait_training_finish(timeout: int, wait: bool, mt_id: str,
mt_client: ModelTrainingClient):
"""
Wait for training to finish according to command line arguments
:param wait:
:param timeout:
:param mt_id: Model Training name
:param mt_client: Model Training Client
"""
if not wait:
return
start = time.time()
if timeout <= 0:
raise Exception(
'Invalid --timeout argument: should be positive integer')
# We create a separate client for logs because it has the different timeout settings
log_mt_client = ModelTrainingClient.construct_from_other(mt_client)
log_mt_client.timeout = mt_client.timeout, LOG_READ_TIMEOUT_SECONDS
click.echo("Logs streaming...")
while True:
elapsed = time.time() - start
if elapsed > timeout:
raise Exception(TIMEOUT_ERROR_MESSAGE)
try:
mt = mt_client.get(mt_id)
if mt.status.state == TRAINING_SUCCESS_STATE:
click.echo(
f'Model {mt_id} was trained. Training took {round(time.time() - start)} seconds'
)
return
elif mt.status.state == TRAINING_FAILED_STATE:
raise Exception(f'Model training {mt_id} was failed.')
elif mt.status.state == "":
click.echo(
f"Can't determine the state of {mt.id}. Sleeping...")
else:
for msg in log_mt_client.log(mt.id, follow=True):
print_logs(msg)
except (WrongHttpStatusCode, HTTPException, RequestException,
APIConnectionException) as e:
LOGGER.info(
'Callback have not confirmed completion of the operation. Exception: %s',
str(e))
LOGGER.debug('Sleep before next request')
time.sleep(DEFAULT_WAIT_TIMEOUT)
