def invalidate(self, request, *args, **kwargs):
endpoint = self.get_object()
socket = Cached(Socket, kwargs={'pk': endpoint.socket_id}).get()
cache_key = ENDPOINT_CACHE_KEY_TEMPLATE.format(
schema=request.instance.pk,
name=endpoint.name,
hash=socket.get_hash(),
)
redis.delete(cache_key)
return HttpResponse(status=status.HTTP_204_NO_CONTENT)
def run_codebox_script(self, request, socket, endpoint, metadata, path, **kwargs):
skip_payload = request._empty_data
# Skip payload if we're dealing with small text/plain requests
# (for backwards compatibility when depending on META.user, META.admin)
uwsgi.add_var('PAYLOAD_PARSED', '0' if skip_payload else '1')
script = Munch(config={
'allow_full_access': True,
'timeout': kwargs.get('timeout', settings.SOCKETS_DEFAULT_TIMEOUT),
'async': kwargs.get('async', settings.SOCKETS_DEFAULT_ASYNC),
'mcpu': kwargs.get('mcpu', settings.SOCKETS_DEFAULT_MCPU),
},
runtime_name=kwargs.get('runtime', LATEST_NODEJS_RUNTIME),
source='')
if path in socket.file_list:
# Prepare spec.
script_files = socket.get_files()
entrypoint = socket.get_local_path(path)
spec = {
'files': script_files,
'source_hash': socket.get_hash(),
'entrypoint': entrypoint,
'output_limit': settings.SOCKETS_MAX_RESULT_SIZE,
'name': endpoint.name,
'cache': kwargs.get('cache', 0),
}
# Add environment
if socket.environment_id:
environment = Cached(SocketEnvironment, kwargs={'pk': socket.environment_id}).get()
if not environment.is_ready:
if environment.status == SocketEnvironment.STATUSES.ERROR:
raise SocketEnvironmentFailure()
raise SocketEnvironmentNotReady()
spec['environment'] = environment.get_hash()
spec['environment_url'] = environment.get_url()
return self.run_view(request, obj=endpoint, script=script, metadata=metadata, endpoint=endpoint,
spec=spec,
skip_payload=skip_payload, flat_args=True,
uwsgi_handler=self.get_codebox_handler)
def process_grpc(self, instance, incentive, spec):
logger = self.get_logger()
from apps.codeboxes.proto import broker_pb2, broker_pb2_grpc
if self.runner is None:
self.channel = grpc.insecure_channel(settings.CODEBOX_BROKER_GRPC,
settings.CODEBOX_GRPC_OPTIONS)
self.runner = broker_pb2_grpc.ScriptRunnerStub(self.channel)
socket = incentive.socket
entrypoint = socket.get_local_path(incentive.codebox.path)
# Add environment
environment_hash = ''
environment_url = ''
if socket.environment_id:
environment = Cached(SocketEnvironment,
kwargs={
'pk': socket.environment_id
}).get()
if not environment.is_ready:
self.block_run('Environment is not yet ready.',
incentive,
instance,
spec,
status=Trace.STATUS_CHOICES.FAILURE)
return
environment_hash = environment.get_hash()
environment_url = environment.get_url()
req = broker_pb2.RunRequest(
meta={
'files': socket.get_files(),
'environmentURL': environment_url,
'trace': json.dumps(spec['trace']).encode(),
'traceID': spec['trace']['id'],
},
lbMeta={
'concurrencyKey': str(instance.pk),
'concurrencyLimit': spec['run']['concurrency_limit'],
},
request=[{
'meta': {
'runtime': spec['run']['runtime_name'],
'sourceHash': socket.get_hash(),
'userID': str(instance.pk),
'environment': environment_hash,
'options': {
'entryPoint': entrypoint,
'outputLimit': settings.CODEBOX_RESULT_SIZE_LIMIT,
'timeout': int(spec['run']['timeout'] * 1000),
'args': spec['run']['additional_args'].encode(),
'config': spec['run']['config'].encode(),
'meta': spec['run']['meta'].encode(),
},
},
}])
# Retry grpc Run if needed.
metadata = create_headers_from_zipkin_attrs(get_tracing_attrs())
if metadata is not None:
metadata = metadata.items()
for i in range(self.grpc_run_retries + 1):
try:
self.runner.Run(req,
timeout=GRPC_RUN_TIMEOUT,
metadata=metadata)
return
except Exception:
if i + 1 > self.grpc_run_retries:
raise
logger.warning("gRPC run failed, retrying (try #%d out of %d)",
i + 1,
self.grpc_run_retries,
exc_info=1)
time.sleep(1)
def process_grpc(self, instance, incentive, spec):
logger = self.get_logger()
if self.runner is None:
tracer_interceptor = client_interceptor.OpenCensusClientInterceptor(
get_current_tracer(),
host_port=settings.CODEBOX_BROKER_GRPC)
channel = grpc.insecure_channel(settings.CODEBOX_BROKER_GRPC)
channel = grpc.intercept_channel(channel, tracer_interceptor)
self.runner = broker_pb2_grpc.ScriptRunnerStub(channel)
socket = incentive.socket
entrypoint = socket.get_local_path(incentive.codebox.path)
# Add environment
environment_hash = ''
environment_url = ''
if socket.environment_id:
environment = Cached(SocketEnvironment, kwargs={'pk': socket.environment_id}).get()
if not environment.is_ready:
self.block_run('Environment is not yet ready.',
incentive, instance, spec,
status=Trace.STATUS_CHOICES.FAILURE)
return
environment_hash = environment.get_hash()
environment_url = environment.get_url()
req = broker_pb2.SimpleRunRequest(
meta={
'files': socket.get_files(),
'environment_url': environment_url,
'trace': json.dumps(spec['trace']).encode(),
'trace_id': spec['trace']['id'],
},
lb_meta={
'concurrency_key': str(instance.pk),
'concurrency_limit': spec['run']['concurrency_limit'],
},
script_meta={
'runtime': spec['run']['runtime_name'],
'source_hash': socket.get_hash(),
'user_id': str(instance.pk),
'environment': environment_hash,
'options': {
'entrypoint': entrypoint,
'output_limit': settings.CODEBOX_RESULT_SIZE_LIMIT,
'timeout': int(spec['run']['timeout'] * 1000),
'async': spec['run']['async'],
'mcpu': spec['run']['mcpu'],
'args': spec['run']['additional_args'].encode(),
'config': spec['run']['config'].encode(),
'meta': spec['run']['meta'].encode(),
},
},
)
# Retry grpc Run if needed.
for i in range(self.grpc_run_retries + 1):
try:
response = self.runner.SimpleRun(req, timeout=GRPC_RUN_TIMEOUT)
for _ in response:
# Drain response so it is processed and not queued
pass
return
except Exception:
if i + 1 > self.grpc_run_retries:
raise
logger.warning("gRPC run failed, retrying (try #%d out of %d)", i + 1, self.grpc_run_retries,
exc_info=1)
time.sleep(1)