def invalidate(self, request, *args, **kwargs): endpoint = self.get_object() socket = Cached(Socket, kwargs={'pk': endpoint.socket_id}).get() cache_key = ENDPOINT_CACHE_KEY_TEMPLATE.format( schema=request.instance.pk, name=endpoint.name, hash=socket.get_hash(), ) redis.delete(cache_key) return HttpResponse(status=status.HTTP_204_NO_CONTENT)
def run_codebox_script(self, request, socket, endpoint, metadata, path, **kwargs): skip_payload = request._empty_data # Skip payload if we're dealing with small text/plain requests # (for backwards compatibility when depending on META.user, META.admin) uwsgi.add_var('PAYLOAD_PARSED', '0' if skip_payload else '1') script = Munch(config={ 'allow_full_access': True, 'timeout': kwargs.get('timeout', settings.SOCKETS_DEFAULT_TIMEOUT), 'async': kwargs.get('async', settings.SOCKETS_DEFAULT_ASYNC), 'mcpu': kwargs.get('mcpu', settings.SOCKETS_DEFAULT_MCPU), }, runtime_name=kwargs.get('runtime', LATEST_NODEJS_RUNTIME), source='') if path in socket.file_list: # Prepare spec. script_files = socket.get_files() entrypoint = socket.get_local_path(path) spec = { 'files': script_files, 'source_hash': socket.get_hash(), 'entrypoint': entrypoint, 'output_limit': settings.SOCKETS_MAX_RESULT_SIZE, 'name': endpoint.name, 'cache': kwargs.get('cache', 0), } # Add environment if socket.environment_id: environment = Cached(SocketEnvironment, kwargs={'pk': socket.environment_id}).get() if not environment.is_ready: if environment.status == SocketEnvironment.STATUSES.ERROR: raise SocketEnvironmentFailure() raise SocketEnvironmentNotReady() spec['environment'] = environment.get_hash() spec['environment_url'] = environment.get_url() return self.run_view(request, obj=endpoint, script=script, metadata=metadata, endpoint=endpoint, spec=spec, skip_payload=skip_payload, flat_args=True, uwsgi_handler=self.get_codebox_handler)
def process_grpc(self, instance, incentive, spec): logger = self.get_logger() from apps.codeboxes.proto import broker_pb2, broker_pb2_grpc if self.runner is None: self.channel = grpc.insecure_channel(settings.CODEBOX_BROKER_GRPC, settings.CODEBOX_GRPC_OPTIONS) self.runner = broker_pb2_grpc.ScriptRunnerStub(self.channel) socket = incentive.socket entrypoint = socket.get_local_path(incentive.codebox.path) # Add environment environment_hash = '' environment_url = '' if socket.environment_id: environment = Cached(SocketEnvironment, kwargs={ 'pk': socket.environment_id }).get() if not environment.is_ready: self.block_run('Environment is not yet ready.', incentive, instance, spec, status=Trace.STATUS_CHOICES.FAILURE) return environment_hash = environment.get_hash() environment_url = environment.get_url() req = broker_pb2.RunRequest( meta={ 'files': socket.get_files(), 'environmentURL': environment_url, 'trace': json.dumps(spec['trace']).encode(), 'traceID': spec['trace']['id'], }, lbMeta={ 'concurrencyKey': str(instance.pk), 'concurrencyLimit': spec['run']['concurrency_limit'], }, request=[{ 'meta': { 'runtime': spec['run']['runtime_name'], 'sourceHash': socket.get_hash(), 'userID': str(instance.pk), 'environment': environment_hash, 'options': { 'entryPoint': entrypoint, 'outputLimit': settings.CODEBOX_RESULT_SIZE_LIMIT, 'timeout': int(spec['run']['timeout'] * 1000), 'args': spec['run']['additional_args'].encode(), 'config': spec['run']['config'].encode(), 'meta': spec['run']['meta'].encode(), }, }, }]) # Retry grpc Run if needed. metadata = create_headers_from_zipkin_attrs(get_tracing_attrs()) if metadata is not None: metadata = metadata.items() for i in range(self.grpc_run_retries + 1): try: self.runner.Run(req, timeout=GRPC_RUN_TIMEOUT, metadata=metadata) return except Exception: if i + 1 > self.grpc_run_retries: raise logger.warning("gRPC run failed, retrying (try #%d out of %d)", i + 1, self.grpc_run_retries, exc_info=1) time.sleep(1)
def process_grpc(self, instance, incentive, spec): logger = self.get_logger() if self.runner is None: tracer_interceptor = client_interceptor.OpenCensusClientInterceptor( get_current_tracer(), host_port=settings.CODEBOX_BROKER_GRPC) channel = grpc.insecure_channel(settings.CODEBOX_BROKER_GRPC) channel = grpc.intercept_channel(channel, tracer_interceptor) self.runner = broker_pb2_grpc.ScriptRunnerStub(channel) socket = incentive.socket entrypoint = socket.get_local_path(incentive.codebox.path) # Add environment environment_hash = '' environment_url = '' if socket.environment_id: environment = Cached(SocketEnvironment, kwargs={'pk': socket.environment_id}).get() if not environment.is_ready: self.block_run('Environment is not yet ready.', incentive, instance, spec, status=Trace.STATUS_CHOICES.FAILURE) return environment_hash = environment.get_hash() environment_url = environment.get_url() req = broker_pb2.SimpleRunRequest( meta={ 'files': socket.get_files(), 'environment_url': environment_url, 'trace': json.dumps(spec['trace']).encode(), 'trace_id': spec['trace']['id'], }, lb_meta={ 'concurrency_key': str(instance.pk), 'concurrency_limit': spec['run']['concurrency_limit'], }, script_meta={ 'runtime': spec['run']['runtime_name'], 'source_hash': socket.get_hash(), 'user_id': str(instance.pk), 'environment': environment_hash, 'options': { 'entrypoint': entrypoint, 'output_limit': settings.CODEBOX_RESULT_SIZE_LIMIT, 'timeout': int(spec['run']['timeout'] * 1000), 'async': spec['run']['async'], 'mcpu': spec['run']['mcpu'], 'args': spec['run']['additional_args'].encode(), 'config': spec['run']['config'].encode(), 'meta': spec['run']['meta'].encode(), }, }, ) # Retry grpc Run if needed. for i in range(self.grpc_run_retries + 1): try: response = self.runner.SimpleRun(req, timeout=GRPC_RUN_TIMEOUT) for _ in response: # Drain response so it is processed and not queued pass return except Exception: if i + 1 > self.grpc_run_retries: raise logger.warning("gRPC run failed, retrying (try #%d out of %d)", i + 1, self.grpc_run_retries, exc_info=1) time.sleep(1)