def test_no_pad_on_too_long():
# arrange
metadata = [{'mimetype': 'text/plain'}] * 10
# act
padded = MetadataFilter.pad_metadata(metadata, 5)
# assert
assert padded == metadata
def test_pad_fills_empty_to_length():
# arrange
metadata = []
# act
padded = MetadataFilter.pad_metadata(metadata, 2)
# assert
assert padded == [{}] * 2
def test_no_pad_on_match_length():
# arrange
metadata = [{'mimetype': 'text/plain'}] * 5
# act
padded = MetadataFilter.pad_metadata(metadata, 5)
# assert
assert padded == metadata
def test_pad_fills_with_none_on_invalid_metadata():
# arrange
metadata = [{'asdf': 'asdf'}]
# act
padded = MetadataFilter.pad_metadata(metadata, 2)
# assert
assert padded == [{}] * 2
def test_pad_fills_to_length():
# arrange
metadata = [{'mimetype': 'text/plain'}]
# act
padded = MetadataFilter.pad_metadata(metadata, 2)
# assert
assert padded == [{'mimetype': 'text/plain'}, {}]
async def fetch_and_scan_all(self, guid, artifact_type, uri, duration,
metadata, chain):
"""Fetch and scan all artifacts concurrently
Args:
guid (str): GUID of the associated bounty
artifact_type (ArtifactType): Artifact type for the bounty being scanned
uri (str): Base artifact URI
duration (int): Max number of blocks to take
metadata (list[dict]) List of metadata json blobs for artifacts
chain (str): Chain we are operating on
Returns:
(list(bool), list(bool), list(str)): Tuple of mask bits, verdicts, and metadatas
"""
async def fetch_and_scan(artifact_metadata, index):
content = await self.client.get_artifact(uri, index)
if not self.bounty_filter.is_allowed(artifact_metadata):
return ScanResult()
if content is not None:
try:
result = await self.scan(
guid, artifact_type,
artifact_type.decode_content(content),
artifact_metadata, chain)
except DecodeError:
result = ScanResult()
if result.bit:
result.confidence = self.confidence_modifier.modify(
artifact_metadata, result.confidence)
return result
return ScanResult()
artifacts = await self.client.list_artifacts(uri)
metadata = MetadataFilter.pad_metadata(metadata, len(artifacts))
return await asyncio.gather(
*[fetch_and_scan(metadata[i], i) for i in range(len(artifacts))])
async def scan(self, guid, artifact_type, uri, duration, metadata, chain):
"""Creates a set of jobs to scan all the artifacts at the given URI that are passed via Redis to workers
Args:
guid (str): GUID of the associated bounty
artifact_type (ArtifactType): Artifact type for the bounty being scanned
uri (str): Base artifact URI
duration (int): number of blocks until scan is due
metadata (list[dict]) List of metadata json blobs for artifacts
chain (str): Chain we are operating on
Returns:
list(ScanResult): List of ScanResult objects
"""
# Ensure we don't wait past the scan duration for one large artifact
timeout = duration - self.time_to_post
logger.info(f'Timeout set to {timeout}')
loop = asyncio.get_event_loop()
num_artifacts = len(await self.client.list_artifacts(uri))
# Fill out metadata to match same number of artifacts
metadata = MetadataFilter.pad_metadata(metadata, num_artifacts)
jobs = []
try:
for i in range(num_artifacts):
if (self.bounty_filter is None or self.bounty_filter.is_allowed(metadata[i])) \
and (self.rate_limiter is None or await self.rate_limiter.use()):
job_uri = f'{self.client.polyswarmd_uri}/artifacts/{uri}/{i}/'
job = JobRequest(polyswarmd_uri=self.client.polyswarmd_uri,
guid=guid,
index=i,
uri=job_uri,
artifact_type=artifact_type.value,
duration=timeout,
metadata=metadata[i],
chain=chain,
ts=int(time.time()))
jobs.append(job)
if jobs:
# Update number of jobs sent
loop.create_task(self._update_job_counter(len(jobs)))
# Send jobs as json string to backend
loop.create_task(
self._send_jobs([json.dumps(job.asdict())
for job in jobs]))
# Send jobs to job processor
future = Future()
key = JOB_RESULTS_FORMAT.format(self.queue, guid, chain)
await self.job_processor.register_jobs(guid, key, jobs, future)
# Age off old result keys
loop.create_task(self._expire_key(key, duration + KEY_TIMEOUT))
# Wait for results from job processor
return await future
except OSError:
logger.exception('Redis connection down')
await self.__reset_redis()
except aioredis.errors.ReplyError:
logger.exception('Redis out of memory')
await self.__reset_redis()
except aioredis.errors.ConnectionForcedCloseError:
logger.exception('Redis connection closed')
await self.__reset_redis()
return []