def test_objects(self): token = RedisEphemeralTokens() assert token.key is not None assert token.redis_key == RedisEphemeralTokens.KEY_EPHEMERAL_TOKENS.format( token.key) assert token.get_state() is None assert token.salt is None assert token.ttl is None assert token.scope is None token = RedisEphemeralTokens.generate( scope=token.get_scope(1, 'experiment', 1)) assert token.get_state() is not None assert token.salt is not None assert token.ttl == conf.get(TTL_EPHEMERAL_TOKEN) assert token.scope == token.get_scope(1, 'experiment', 1) assert token.check_token('foo') is False # Checking delete the token assert token.get_state() is None token = RedisEphemeralTokens.generate( scope=token.get_scope(1, 'experiment', 1)) assert token.check_token(None) is False # Checking delete the token assert token.get_state() is None token = RedisEphemeralTokens.generate( scope=token.get_scope(1, 'experiment', 1)) valid = RedisEphemeralTokens.make_token(token) assert token.check_token(valid) is True # Checking delete the token assert token.get_state() is None assert token.salt is None assert token.ttl is None assert token.scope is None
def post(self, request, *args, **kwargs): user = request.user if user.scope is None: return Response(status=status.HTTP_403_FORBIDDEN) experiment = self.get_object() if experiment.last_status not in [ExperimentLifeCycle.SCHEDULED, ExperimentLifeCycle.STARTING, ExperimentLifeCycle.RUNNING]: return Response(status=status.HTTP_403_FORBIDDEN) scope = RedisEphemeralTokens.get_scope(user=experiment.user.id, model='experiment', object_id=experiment.id) if sorted(user.scope) != sorted(scope): return Response(status=status.HTTP_403_FORBIDDEN) token, _ = Token.objects.get_or_create(user=experiment.user) return Response({'token': token.key}, status=status.HTTP_200_OK)
def start_experiment(experiment): # Update experiment status to show that its started experiment.set_status(ExperimentLifeCycle.SCHEDULED) project = experiment.project group = experiment.experiment_group job_docker_image = None # This will force the spawners to use the default docker image if experiment.specification.build: try: image_name, image_tag = get_image_info( build_job=experiment.build_job) except (ValueError, AttributeError): _logger.error('Could not start the experiment.', exc_info=True) experiment.set_status(ExperimentLifeCycle.FAILED, message='Image info was not found.') return job_docker_image = '{}:{}'.format(image_name, image_tag) _logger.info('Start experiment with built image `%s`', job_docker_image) else: _logger.info('Start experiment with default image.') spawner_class = get_spawner_class(experiment.specification.framework) token_scope = RedisEphemeralTokens.get_scope(experiment.user.id, 'experiment', experiment.id) error = {} try: # Use spawners to start the experiment spawner = spawner_class( project_name=project.unique_name, experiment_name=experiment.unique_name, experiment_group_name=group.unique_name if group else None, project_uuid=project.uuid.hex, experiment_group_uuid=group.uuid.hex if group else None, experiment_uuid=experiment.uuid.hex, persistence_config=experiment.persistence_config, outputs_refs_experiments=experiment.outputs_refs_experiments, outputs_refs_jobs=experiment.outputs_refs_jobs, original_name=experiment.original_unique_name, cloning_strategy=experiment.cloning_strategy, spec=experiment.specification, k8s_config=conf.get('K8S_CONFIG'), namespace=conf.get('K8S_NAMESPACE'), in_cluster=True, job_docker_image=job_docker_image, use_sidecar=True, token_scope=token_scope) response = spawner.start_experiment() handle_experiment(experiment=experiment, spawner=spawner, response=response) except ApiException as e: _logger.error( 'Could not start the experiment, please check your polyaxon spec.', exc_info=True) error = { 'raised': True, 'traceback': traceback.format_exc(), 'message': 'Could not start the experiment, encountered a Kubernetes ApiException.' } except VolumeNotFoundError as e: _logger.error( 'Could not start the experiment, please check your volume definitions.', exc_info=True) error = { 'raised': True, 'traceback': traceback.format_exc(), 'message': 'Could not start the experiment, ' 'encountered a volume definition problem, %s.' % e } except Exception as e: _logger.error( 'Could not start the experiment, please check your polyaxon spec', exc_info=True) error = { 'raised': True, 'traceback': traceback.format_exc(), 'message': 'Could not start the experiment encountered an {} exception.'. format(e.__class__.__name__) } finally: if error.get('raised'): experiment.set_status(ExperimentLifeCycle.FAILED, message=error.get('message'), traceback=error.get('traceback'))