async def _get_job(self, command, socket):
''' Get status for a single job. '''
job_id = str(UUID(bytes=command.job_id))
job_doc = await self._crawl_manager.get_job(job_id)
response = Response()
if job_doc is None:
response.is_success = False
response.error_message = f'No job exists with ID={job_id}'
else:
job = response.job
job.job_id = UUID(job_doc['id']).bytes
for seed in job_doc['seeds']:
job.seeds.append(seed)
for tag in job_doc['tags']:
job.tag_list.tags.append(tag)
Policy.convert_doc_to_pb(job_doc['policy'], job.policy)
job.name = job_doc['name']
job.item_count = job_doc['item_count']
job.http_success_count = job_doc['http_success_count']
job.http_error_count = job_doc['http_error_count']
job.exception_count = job_doc['exception_count']
job.started_at = job_doc['started_at'].isoformat()
if job_doc['completed_at'] is not None:
job.completed_at = job_doc['completed_at'].isoformat()
run_state = job_doc['run_state'].upper()
job.run_state = protobuf.shared_pb2.JobRunState.Value(run_state)
http_status_counts = job_doc['http_status_counts']
for status_code, count in http_status_counts.items():
job.http_status_counts[int(status_code)] = count
return response
async def _get_job_schedule(self, command, socket):
''' Get metadata for a job schedule. '''
schedule_id = str(UUID(bytes=command.schedule_id))
doc = await self._scheduler.get_job_schedule(schedule_id)
response = Response()
if doc is None:
response.is_success = False
response.error_message = f'No schedule exists with ID={schedule_id}'
else:
pb = response.job_schedule
Scheduler.doc_to_pb(doc, pb)
return response
async def _handle_request(self, client_ip, websocket, request_data):
''' Handle a single request/response pair. '''
request = Request.FromString(request_data)
start = time()
try:
command_name = request.WhichOneof('Command')
if command_name is None:
raise InvalidRequestException('No command specified')
command = getattr(request, command_name)
try:
handler = self._request_handlers[command_name]
except KeyError:
raise InvalidRequestException(
'Invalid command name: {}'.format(command_name))
response = await handler(command, websocket)
response.request_id = request.request_id
response.is_success = True
elapsed = time() - start
logger.info('Request OK %s %s %0.3fs', command_name, client_ip,
elapsed)
except asyncio.CancelledError:
raise
except Exception as e:
if isinstance(e, InvalidRequestException):
elapsed = time() - start
logger.error('Request ERROR %s %s %0.3fs', command_name,
client_ip, elapsed)
else:
logger.exception('Exception while handling request:\n%r',
request)
response = Response()
response.is_success = False
response.error_message = str(e)
try:
response.request_id = request.request_id
except:
# A parsing failure could lead to request or request_id not
# being defined. There's nothing we can do to fix this.
pass
if response.IsInitialized():
message = ServerMessage()
message.response.MergeFrom(response)
message_data = message.SerializeToString()
await websocket.send(message_data)
else:
# This could happen, e.g. if the request_id is not set.
logger.error('Cannot send uninitialized response:\n%r', response)