async def respond(self, job: JobRequest, response: JobResponse): logger.info('Scan results for job %s', job.key, extra={'extra': response.asdict()}) key = f'{self.queue}_{job.guid}_{job.chain}_results' with await self.redis as redis: await redis.rpush(key, json.dumps(response.asdict()))
async def respond(self, job: JobRequest, response: JobResponse): try: logger.info('Scan results for job %s', job.key, extra={'extra': response.asdict()}) key = f'{self.queue}_{job.guid}_{job.chain}_results' await self.redis.rpush(key, json.dumps(response.asdict())) except OSError: logger.exception('Error pushing results for %s', job.guid) raise
async def test_results_after_timeout(redis_client, job_processor): # Add response before registering job_response = JobResponse(index=0, bit=True, verdict=False, confidence=.5, metadata='') await redis_client.rpush('test_results_after_timeout', json.dumps(job_response.asdict())) jobs = [JobRequest('polyswarmd-addr', 'guid', i, 'uri', ArtifactType.FILE.value, 1, None, 'side', int(time.time())) for i in range(1)] future = Future() await job_processor.register_jobs('guid', 'test_results_after_timeout', jobs, future) scan_results = await future assert len(scan_results) == 1 assert scan_results[0].bit
async def test_results_after_complete(redis_client, job_processor): job = JobRequest('polyswarmd-addr', 'guid', 0, 'uri', ArtifactType.FILE.value, 10, None, 'side', int(time.time())) future = Future() await job_processor.register_jobs('guid', 'test_results_after_complete', [job], future) # Add response before waiting on the future job_response = JobResponse(index=0, bit=True, verdict=False, confidence=.5, metadata='') await redis_client.rpush('test_results_after_complete', json.dumps(job_response.asdict())) scan_results = await future assert scan_results assert scan_results[0].bit assert not scan_results[0].verdict assert scan_results[0].confidence == .5
async def test_result_values_in_redis(redis_client, job_processor): job = JobRequest('polyswarmd-addr', 'guid', 0, 'uri', ArtifactType.FILE.value, 10, None, 'side', int(time.time())) future = Future() await job_processor.register_jobs('guid', 'test_result_values_in_redis', [job], future) # Add response before waiting on the future job_response = JobResponse(index=0, bit=True, verdict=False, confidence=.5, metadata='') await asyncio.sleep(1) await redis_client.rpush('test_result_values_in_redis', json.dumps(job_response.asdict())) await future # wait for the loop to finish in job_processor await asyncio.sleep(1) assert float(await redis_client.get('QUEUE_job_completion_time_accum')) > 0 assert int(await redis_client.get('QUEUE_job_completion_times_count')) == 1 assert 0 < float(await redis_client.get('QUEUE_job_completion_time_ratios_accum')) < 1 assert int(await redis_client.get('QUEUE_job_completion_time_ratios_count')) == 1 assert int(await redis_client.get('QUEUE_scan_result_counter')) == 1
async def test_results_thousands_pending_jobs(redis_client, job_processor): count = 10_000 futures = [] def create_job(i): return JobRequest('polyswarmd-addr', f'guid:{i}', 0, 'uri', ArtifactType.FILE.value, 19, None, 'side', int(time.time())) def create_future(): future = Future() futures.append(future) return future await asyncio.gather(*[job_processor.register_jobs(f'guid:{i}', f'test_results_thousands_pending_jobs:{i}', [create_job(i)], create_future()) for i in range(count)]) job_response = JobResponse(index=0, bit=True, verdict=False, confidence=.5, metadata='') await asyncio.gather(*[redis_client.rpush(f'test_results_thousands_pending_jobs:{i}', json.dumps(job_response.asdict())) for i in range(count)]) aggregated_scan_results = await asyncio.gather(*futures) assert len(aggregated_scan_results) == count assert all([scan_results[0].bit for scan_results in aggregated_scan_results])