def test_objects(self):
token = RedisEphemeralTokens()
assert token.key is not None
assert token.redis_key == RedisEphemeralTokens.KEY_EPHEMERAL_TOKENS.format(
token.key)
assert token.get_state() is None
assert token.salt is None
assert token.ttl is None
assert token.scope is None
token = RedisEphemeralTokens.generate(
scope=token.get_scope(1, 'experiment', 1))
assert token.get_state() is not None
assert token.salt is not None
assert token.ttl == conf.get(TTL_EPHEMERAL_TOKEN)
assert token.scope == token.get_scope(1, 'experiment', 1)
assert token.check_token('foo') is False
# Checking delete the token
assert token.get_state() is None
token = RedisEphemeralTokens.generate(
scope=token.get_scope(1, 'experiment', 1))
assert token.check_token(None) is False
# Checking delete the token
assert token.get_state() is None
token = RedisEphemeralTokens.generate(
scope=token.get_scope(1, 'experiment', 1))
valid = RedisEphemeralTokens.make_token(token)
assert token.check_token(valid) is True
# Checking delete the token
assert token.get_state() is None
assert token.salt is None
assert token.ttl is None
assert token.scope is None
def post(self, request, *args, **kwargs):
user = request.user
if user.scope is None:
return Response(status=status.HTTP_403_FORBIDDEN)
experiment = self.get_object()
if experiment.last_status not in [ExperimentLifeCycle.SCHEDULED,
ExperimentLifeCycle.STARTING,
ExperimentLifeCycle.RUNNING]:
return Response(status=status.HTTP_403_FORBIDDEN)
scope = RedisEphemeralTokens.get_scope(user=experiment.user.id,
model='experiment',
object_id=experiment.id)
if sorted(user.scope) != sorted(scope):
return Response(status=status.HTTP_403_FORBIDDEN)
token, _ = Token.objects.get_or_create(user=experiment.user)
return Response({'token': token.key}, status=status.HTTP_200_OK)
def start_experiment(experiment):
# Update experiment status to show that its started
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
project = experiment.project
group = experiment.experiment_group
job_docker_image = None # This will force the spawners to use the default docker image
if experiment.specification.build:
try:
image_name, image_tag = get_image_info(
build_job=experiment.build_job)
except (ValueError, AttributeError):
_logger.error('Could not start the experiment.', exc_info=True)
experiment.set_status(ExperimentLifeCycle.FAILED,
message='Image info was not found.')
return
job_docker_image = '{}:{}'.format(image_name, image_tag)
_logger.info('Start experiment with built image `%s`',
job_docker_image)
else:
_logger.info('Start experiment with default image.')
spawner_class = get_spawner_class(experiment.specification.framework)
token_scope = RedisEphemeralTokens.get_scope(experiment.user.id,
'experiment', experiment.id)
error = {}
try:
# Use spawners to start the experiment
spawner = spawner_class(
project_name=project.unique_name,
experiment_name=experiment.unique_name,
experiment_group_name=group.unique_name if group else None,
project_uuid=project.uuid.hex,
experiment_group_uuid=group.uuid.hex if group else None,
experiment_uuid=experiment.uuid.hex,
persistence_config=experiment.persistence_config,
outputs_refs_experiments=experiment.outputs_refs_experiments,
outputs_refs_jobs=experiment.outputs_refs_jobs,
original_name=experiment.original_unique_name,
cloning_strategy=experiment.cloning_strategy,
spec=experiment.specification,
k8s_config=conf.get('K8S_CONFIG'),
namespace=conf.get('K8S_NAMESPACE'),
in_cluster=True,
job_docker_image=job_docker_image,
use_sidecar=True,
token_scope=token_scope)
response = spawner.start_experiment()
handle_experiment(experiment=experiment,
spawner=spawner,
response=response)
except ApiException as e:
_logger.error(
'Could not start the experiment, please check your polyaxon spec.',
exc_info=True)
error = {
'raised':
True,
'traceback':
traceback.format_exc(),
'message':
'Could not start the experiment, encountered a Kubernetes ApiException.'
}
except VolumeNotFoundError as e:
_logger.error(
'Could not start the experiment, please check your volume definitions.',
exc_info=True)
error = {
'raised':
True,
'traceback':
traceback.format_exc(),
'message':
'Could not start the experiment, '
'encountered a volume definition problem, %s.' % e
}
except Exception as e:
_logger.error(
'Could not start the experiment, please check your polyaxon spec',
exc_info=True)
error = {
'raised':
True,
'traceback':
traceback.format_exc(),
'message':
'Could not start the experiment encountered an {} exception.'.
format(e.__class__.__name__)
}
finally:
if error.get('raised'):
experiment.set_status(ExperimentLifeCycle.FAILED,
message=error.get('message'),
traceback=error.get('traceback'))