Esempio n. 1
0
def bounty_to_dict(bounty):
    return {
        'guid': str(uuid.UUID(int=bounty[0])),
        'artifact_type': ArtifactType.to_string(ArtifactType(bounty[1])),
        'author': bounty[2],
        'amount': str(bounty[3]),
        'uri': bounty[4],
        'num_artifacts': bounty[5],
        'expiration': bounty[6],
        'assigned_arbiter': bounty[7],
        'quorum_reached': bounty[8],
        'quorum_reached_block': bounty[9],
        'quorum_mask': safe_int_to_bool_list(bounty[10], bounty[5]),
        'metadata': bounty[11]
    }
Esempio n. 2
0
    async def post_bounty(self,
                          artifact_type,
                          amount,
                          artifact_uri,
                          duration,
                          chain,
                          api_key=None,
                          metadata=None):
        """Post a bounty to polyswarmd.

        Args:
            artifact_type (ArtifactType): The artifact type in this bounty
            amount (int): The amount to put up as a bounty
            artifact_uri (str): URI of artifacts
            duration (int): Number of blocks to accept new assertions
            chain (str): Which chain to operate on
            api_key (str): Override default API key
            metadata (Optional[str]): Optional metadata
        Returns:
            Response JSON parsed from polyswarmd containing emitted events
        """
        bounty_fee = await self.parameters[chain].get('bounty_fee')
        bloom = await self.calculate_bloom(artifact_uri)
        num_artifacts = await self.__client.get_artifact_count(artifact_uri)
        transaction = PostBountyTransaction(
            self.__client, ArtifactType.to_string(artifact_type), amount,
            bounty_fee, artifact_uri, num_artifacts, duration, bloom, metadata)
        success, result = await transaction.send(chain, api_key=api_key)

        if not success or 'bounties' not in result:
            logger.error('Expected bounty, received', extra={'extra': result})

        return result.get('bounties', [])
Esempio n. 3
0
        def collect_metrics(scan: 'ScanResult', start: 'float', artifact_type: 'ArtifactType'):
            """Collect application metrics from this scan"""
            # Collect timing information
            statsd.timing(SCAN_TIME, perf_counter() - start)

            type_tag = 'type:%s' % ArtifactType.to_string(artifact_type)

            if scan.bit is True:
                if scan.verdict is True:
                    verdict_tag = 'verdict:malicious'
                elif scan.verdict is False:
                    verdict_tag = 'verdict:benign'
                elif scan.verdict is None:
                    verdict_tag = 'verdict:none'
                else:
                    verdict_tag = 'verdict:invalid.%s' % type(scan.verdict).__name__

                if verbose:
                    statsd.increment(SCAN_VERDICT, tags=[type_tag, verdict_tag])

                statsd.increment(SCAN_SUCCESS, tags=[type_tag, verdict_tag])

            elif scan.bit is False:
                try:
                    # Treat any scan result w/ bit=False & 'scan_error' in metadata as an error
                    statsd.increment(SCAN_FAIL, tags=[
                        type_tag,
                        'scan_error:%s' % extract_verdict(scan).__dict__['scan_error']
                    ])
                except (AttributeError, KeyError):
                    # otherwise, the engine is just reporting no result
                    statsd.increment(SCAN_NO_RESULT, tags=[type_tag])

            else:
                statsd.increment(SCAN_TYPE_INVALID, tags=[type_tag])
Esempio n. 4
0
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
         testing, insecure_transport, chains, log_format, artifact_type):
    """
    Entrypoint for the arbiter driver
    """
    loglevel = getattr(logging, log.upper(), None)
    clientlevel = getattr(logging, client_log.upper(), None)
    if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
        logging.error('invalid log level')
        sys.exit(-1)

    logger_name, arbiter_class = choose_backend(backend)

    init_logging(['arbiter', logger_name], log_format, loglevel)
    init_logging(['polyswarmclient'], log_format, clientlevel)

    artifact_types = None
    if artifact_type:
        artifact_types = [
            ArtifactType.from_string(artifact) for artifact in artifact_type
        ]

    arbiter_class.connect(polyswarmd_addr,
                          keyfile,
                          password,
                          api_key=api_key,
                          testing=testing,
                          insecure_transport=insecure_transport,
                          chains=set(chains),
                          artifact_types=artifact_types).run()
Esempio n. 5
0
    def verify(self, transaction):
        try:
            decoded = DecodedTransaction.from_transaction(transaction, self.ABI)
        except ValueError as e:
            logger.error('Transaction verification failed: %s', str(e))
            return False

        logger.debug('Expected: %s, Actual: %s', self, decoded)
        guid, artifact_type, amount, artifact_uri, num_artifacts, duration, bloom, metadata = decoded.parameters

        bloom_value = 0
        for b in bloom:
            bloom_value = bloom_value << 256 | int(b)

        artifact_type = ArtifactType(int(artifact_type))
        artifact_uri = artifact_uri.decode('utf-8')

        return decoded.value == 0 and \
            artifact_type == self.artifact_type and \
            artifact_uri == self.artifact_uri and \
            num_artifacts == self.num_artifacts and \
            duration == self.duration and \
            bloom_value == self.bloom and \
            amount == self.amount and \
            metadata.decode('utf-8') == self.metadata
Esempio n. 6
0
def test_scanalytics(statsd, engine_info, use_async, scan_result, verbose_metrics, artifact_kind):
    is_error = isinstance(scan_result, Exception)
    args = (None, str(uuid4()), artifact_kind, b'content', {}, 'home')
    type_tag = 'type:%s' % ArtifactType.to_string(artifact_kind)
    if use_async:

        @scanalytics(statsd=statsd, engine_info=engine_info, verbose=verbose_metrics)
        async def scanfn(self, guid, artifact_type, content, metadata, chain):
            if is_error:
                raise scan_result
            return scan_result

        result = asyncio.run(scanfn(*args))
    else:

        @scanalytics(statsd=statsd, engine_info=engine_info, verbose=verbose_metrics)
        def scanfn(self, guid, artifact_type, content, metadata, chain):
            if is_error:
                raise scan_result
            return scan_result

        result = scanfn(*args)

    statsd.timing.assert_called_once()

    assert isinstance(result.metadata, str)
    result_meta = Verdict.parse_raw(result.metadata)
    assert result_meta.scanner.signatures_version == engine_info.definitions_version
    assert result_meta.scanner.vendor_version == engine_info.engine_version

    if is_error:
        assert result_meta.__dict__['scan_error'] == scan_result.event_name
        assert result.bit is False
        statsd.increment.assert_called_once_with(
            SCAN_FAIL, tags=[type_tag, f'scan_error:{scan_result.event_name}']
        )
    else:
        assert result.verdict is scan_result.verdict
        assert result.bit is scan_result.bit
        verdict_tag = 'verdict:malicious' if scan_result.verdict else 'verdict:benign'

        if scan_result.bit is True:
            statsd.increment.assert_any_call(SCAN_SUCCESS, tags=[type_tag, verdict_tag])

            if verbose_metrics:
                statsd.increment.assert_any_call(
                    SCAN_VERDICT,
                    tags=[type_tag, verdict_tag],
                )
                assert statsd.increment.call_count == 2
            else:
                assert statsd.increment.call_count == 1

        elif scan_result.bit is False:
            if verbose_metrics:
                statsd.increment.assert_called_once_with(SCAN_NO_RESULT, tags=[type_tag])
Esempio n. 7
0
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
         testing, insecure_transport, allow_key_over_http, chains, log_format,
         artifact_type, bid_strategy, accept, exclude, filter, confidence):
    """ Entrypoint for the microengine driver
    """
    utils.fast_deprecation()
    loglevel = getattr(logging, log.upper(), None)
    clientlevel = getattr(logging, client_log.upper(), None)
    if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
        logging.error('invalid log level')
        raise FatalError('Invalid log level', 1)

    polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key,
                                                     allow_key_over_http,
                                                     insecure_transport)
    if insecure_transport:
        warnings.warn(
            '--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`',
            DeprecationWarning)

    logger_name, microengine_class = choose_backend(backend)
    bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy)

    artifact_types = None
    init_logging(['microengine', logger_name], log_format, loglevel)
    init_logging(['polyswarmclient'], log_format, clientlevel)

    if artifact_type:
        artifact_types = [
            ArtifactType.from_string(artifact) for artifact in artifact_type
        ]

    filter_accept = filter.get('accept', [])
    filter_reject = filter.get('reject', [])
    if accept or exclude:
        warnings.warn(
            'Options `--exclude|accept key:value` are deprecated, please switch to'
            ' `--filter accept|reject key comparison value`',
            DeprecationWarning)
        filter_accept.extend(accept)
        filter_reject.extend(exclude)

    favor = confidence.get('favor', [])
    penalize = confidence.get('penalize', [])

    microengine_class.connect(
        polyswarmd_addr,
        keyfile,
        password,
        api_key=api_key,
        artifact_types=artifact_types,
        bid_strategy=bid_strategy_class(),
        bounty_filter=BountyFilter(filter_accept, filter_reject),
        chains=set(chains),
        confidence_modifier=ConfidenceModifier(favor, penalize),
        testing=testing).run()
Esempio n. 8
0
    def __init__(self, artifact_type, amount, artifact_uri, num_artifacts, duration, bloom, metadata):
        super().__init__((UNKNOWN_PARAMETER, amount, artifact_uri, num_artifacts, duration, bloom, metadata))

        self.artifact_type = ArtifactType.from_string(artifact_type)
        self.amount = amount
        self.artifact_uri = artifact_uri
        self.num_artifacts = num_artifacts
        self.duration = duration
        self.bloom = bloom
        self.metadata = metadata
Esempio n. 9
0
 def make_bounty():
     return {
         'guid': str(uuid.uuid4()),
         'artifact_type': ArtifactType.to_string(ArtifactType.FILE),
         'author': random_address(),
         'amount': random.randint(0, 10**18),
         'uri': random_ipfs_uri(),
         'expiration': random.randint(0, 1000),
         'metadata': None,
     }
Esempio n. 10
0
def post_bounties():
    config = app.config['POLYSWARMD']
    session = app.config['REQUESTS_SESSION']
    account = g.chain.w3.toChecksumAddress(g.eth_address)
    base_nonce = int(
        request.args.get('base_nonce',
                         g.chain.w3.eth.getTransactionCount(account)))

    body = request.get_json()
    try:
        _post_bounties_schema(body)
    except fastjsonschema.JsonSchemaException as e:
        return failure('Invalid JSON: ' + e.message, 400)

    guid = uuid.uuid4()
    artifact_type = ArtifactType.from_string(body['artifact_type'])
    amount = int(body['amount'])
    artifact_uri = body['uri']
    duration_blocks = body['duration']
    metadata = body.get('metadata', '')

    try:
        arts = config.artifact.client.ls(artifact_uri, session)
    except HTTPError as e:
        return failure(e.response.content, e.response.status_code)
    except ArtifactException:
        logger.exception('Failed to ls given artifact uri')
        return failure(f'Failed to check artifact uri', 500)

    if amount < eth.bounty_amount_min(
            g.chain.bounty_registry.contract) * len(arts):
        return failure('Invalid bounty amount', 400)

    if metadata and not config.artifact.client.check_uri(metadata):
        return failure('Invalid bounty metadata URI (should be IPFS hash)',
                       400)

    num_artifacts = len(arts)
    bloom = calculate_bloom(arts)

    approve_amount = amount + eth.bounty_fee(g.chain.bounty_registry.contract)

    transactions = [
        build_transaction(
            g.chain.nectar_token.contract.functions.approve(
                g.chain.bounty_registry.contract.address, approve_amount),
            base_nonce),
        build_transaction(
            g.chain.bounty_registry.contract.functions.postBounty(
                guid.int, artifact_type.value, amount, artifact_uri,
                num_artifacts, duration_blocks, bloom, metadata),
            base_nonce + 1),
    ]

    return success({'transactions': transactions})
Esempio n. 11
0
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
         testing, insecure_transport, chains, log_format, artifact_type,
         bid_strategy, accept, exclude):
    """Entrypoint for the microengine driver

    Args:
        log (str): Logging level for all app logs
        client_log (str): Logging level for all polyswarmclient logs
        polyswarmd_addr(str): Address of polyswarmd
        keyfile (str): Path to private key file to use to sign transactions
        password (str): Password to decrypt the encrypted private key
        backend (str): Backend implementation to use
        api_key(str): API key to use with polyswarmd
        testing (int): Mode to process N bounties then exit (optional)
        insecure_transport (bool): Connect to polyswarmd without TLS
        chains (list[str]): List of chains on which to scan artifacts
        log_format (str): Format to output logs in. `text` or `json`
        artifact_type (list[str]): List of artifact types to scan
        bid_strategy (str): Bid strategy module name
        accept (list[tuple[str]]): List of excluded mimetypes
        exclude (list[tuple[str]]): List of excluded mimetypes
    """
    loglevel = getattr(logging, log.upper(), None)
    clientlevel = getattr(logging, client_log.upper(), None)
    if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
        logging.error('invalid log level')
        sys.exit(-1)

    logger_name, microengine_class = choose_backend(backend)
    bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy)

    artifact_types = None
    init_logging(['microengine', logger_name], log_format, loglevel)
    init_logging(['polyswarmclient'], log_format, clientlevel)

    if artifact_type:
        artifact_types = [
            ArtifactType.from_string(artifact) for artifact in artifact_type
        ]

    microengine_class.connect(polyswarmd_addr,
                              keyfile,
                              password,
                              api_key=api_key,
                              testing=testing,
                              insecure_transport=insecure_transport,
                              chains=set(chains),
                              artifact_types=artifact_types,
                              exclude=exclude,
                              accept=accept,
                              bid_strategy=bid_strategy_class()).run()
Esempio n. 12
0
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
         testing, insecure_transport, chains, log_format, artifact_type,
         bid_strategy, accept, exclude, filter, confidence):
    """Entrypoint for the microengine driver
    """
    loglevel = getattr(logging, log.upper(), None)
    clientlevel = getattr(logging, client_log.upper(), None)
    if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
        logging.error('invalid log level')
        sys.exit(-1)

    logger_name, microengine_class = choose_backend(backend)
    bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy)

    artifact_types = None
    init_logging(['microengine', logger_name], log_format, loglevel)
    init_logging(['polyswarmclient'], log_format, clientlevel)

    if artifact_type:
        artifact_types = [
            ArtifactType.from_string(artifact) for artifact in artifact_type
        ]

    filter_accept = filter.get('accept', [])
    filter_reject = filter.get('reject', [])
    if accept or exclude:
        logger.warning(
            'Options `--exclude|accept key:value` are deprecated, please switch to `--filter '
            'accept|reject key comparison value`')

        filter_accept.extend(accept)
        filter_reject.extend(exclude)

    favor = confidence.get('favor', [])
    penalize = confidence.get('penalize', [])

    microengine_class.connect(
        polyswarmd_addr,
        keyfile,
        password,
        api_key=api_key,
        artifact_types=artifact_types,
        bid_strategy=bid_strategy_class(),
        bounty_filter=BountyFilter(filter_accept, filter_reject),
        chains=set(chains),
        confidence_modifier=ConfidenceModifier(favor, penalize),
        insecure_transport=insecure_transport,
        testing=testing).run()
Esempio n. 13
0
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
         testing, insecure_transport, chains, log_format, artifact_type):
    """Entrypoint for the arbiter driver

    Args:
        log (str): Logging level for all app logs
        client_log (str): Logging level for all polyswarmclient logs
        polyswarmd_addr(str): Address of polyswarmd
        keyfile (str): Path to private key file to use to sign transactions
        password (str): Password to decrypt the encrypted private key
        backend (str): Backend implementation to use
        api_key(str): API key to use with polyswarmd
        testing (int): Mode to process N bounties then exit (optional)
        insecure_transport (bool): Connect to polyswarmd without TLS
        chains (List[str]): Chain(s) to operate on
        log_format (str): Format to output logs in. `text` or `json`
        artifact_type (list[str]): List of artifact types to scan
    """

    loglevel = getattr(logging, log.upper(), None)
    clientlevel = getattr(logging, client_log.upper(), None)
    if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
        logging.error('invalid log level')
        sys.exit(-1)

    logger_name, arbiter_class = choose_backend(backend)

    init_logging(['arbiter', logger_name], log_format, loglevel)
    init_logging(['polyswarmclient'], log_format, clientlevel)

    artifact_types = None
    if artifact_type:
        artifact_types = [
            ArtifactType.from_string(artifact) for artifact in artifact_type
        ]

    arbiter_class.connect(polyswarmd_addr,
                          keyfile,
                          password,
                          api_key=api_key,
                          testing=testing,
                          insecure_transport=insecure_transport,
                          chains=set(chains),
                          artifact_types=artifact_types).run()
Esempio n. 14
0
    async def run(self, guid, artifact_type, author, amount, uri, expiration,
                  metadata, block_number, txhash, chain):
        """Run the registered callbacks

        Args:
            guid (str): Bounty GUID
            artifact_type (str): String representation of artifact type
            author (str): Author of the bounty
            amount (str): Bounty reward amount
            uri (str): URI of the artifacts in the bounty
            expiration (int): Block number the bounty expires on
            metadata (dict): Dictionary or string of metadata
            block_number (int): Block number the bounty was posted on
            txhash (str): Transaction hash which caused the event
            chain (str): Chain event received on
        """
        return await super().run(guid, ArtifactType.from_string(artifact_type),
                                 author, amount, uri, expiration, metadata,
                                 block_number, txhash, chain)
Esempio n. 15
0
    async def handle_new_bounty(guid, artifact_type, author, amount, uri,
                                expiration, metadata, block_number, txhash,
                                chain):
        new_bounty = {
            'guid': guid,
            'artifact_type': ArtifactType.to_string(artifact_type),
            'author': author,
            'amount': amount,
            'uri': uri,
            'expiration': expiration,
            'metadata': metadata,
        }

        if chain == 'home':
            assert new_bounty == home_bounty
            home_done.set()
        elif chain == 'side':
            assert new_bounty == side_bounty
            side_done.set()
        else:
            raise ValueError('Invalid chain')
Esempio n. 16
0
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
         testing, insecure_transport, allow_key_over_http, chains, log_format,
         artifact_type):
    """
    Entrypoint for the arbiter driver
    """
    utils.fast_deprecation()

    loglevel = getattr(logging, log.upper(), None)
    clientlevel = getattr(logging, client_log.upper(), None)
    if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
        logging.error('invalid log level')
        raise FatalError('Invalid log level', 1)

    logger_name, arbiter_class = choose_backend(backend)

    init_logging(['arbiter', logger_name], log_format, loglevel)
    init_logging(['polyswarmclient'], log_format, clientlevel)

    polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key,
                                                     allow_key_over_http,
                                                     insecure_transport)
    if insecure_transport:
        warnings.warn(
            '--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`',
            DeprecationWarning)

    artifact_types = None
    if artifact_type:
        artifact_types = [
            ArtifactType.from_string(artifact) for artifact in artifact_type
        ]

    arbiter_class.connect(polyswarmd_addr,
                          keyfile,
                          password,
                          api_key=api_key,
                          testing=testing,
                          chains=set(chains),
                          artifact_types=artifact_types).run()
Esempio n. 17
0
    def __post_init__(self):
        if not BountyMetadata.validate(self.metadata):
            raise ValueError

        if not ArtifactType(self.artifact_type):
            raise ValueError
def test_url_artifact_type_from_uppercase_string():
    # arrange
    # act
    artifact_type = ArtifactType.from_string('URL')
    # assert
    assert artifact_type == ArtifactType.URL
def test_file_artifact_type_from_lowercase_string():
    # arrange
    # act
    artifact_type = ArtifactType.from_string('file')
    # assert
    assert artifact_type == ArtifactType.FILE
def test_file_artifact_type_to_string():
    # arrange
    # act
    # assert
    assert ArtifactType.to_string(ArtifactType.FILE) == 'file'
def test_url_artifact_type_to_string():
    # arrange
    # act
    # assert
    assert ArtifactType.to_string(ArtifactType.URL) == 'url'
def test_url_artifact_type_from_int():
    # arrange
    # act
    artifact_type = ArtifactType(1)
    # assert
    assert artifact_type == ArtifactType.URL
def test_file_artifact_type_from_int():
    # arrange
    # act
    artifact_type = ArtifactType(0)
    # assert
    assert artifact_type == ArtifactType.FILE
Esempio n. 24
0
class NewBounty(WebsocketFilterMessage[NewBountyMessageData]):
    """NewBounty

    doctest:
    When the doctest runs, MetadataHandler._substitute_metadata is already defined outside this
    doctest (in __main__.py). Running this as a doctest will not trigger network IO.

    >>> event = mkevent({
    ... 'guid': 1066,
    ... 'artifactType': 1,
    ... 'author': addr1,
    ... 'amount': 10,
    ... 'artifactURI': '912bnadf01295',
    ... 'expirationBlock': 118,
    ... 'metadata': 'ZWassertionuri'})
    >>> decoded_msg(NewBounty.serialize_message(event))
    {'block_number': 117,
     'data': {'amount': '10',
              'artifact_type': 'url',
              'author': '0x00000000000000000000000000000001',
              'expiration': '118',
              'guid': '00000000-0000-0000-0000-00000000042a',
              'metadata': [{'bounty_id': 69540800813340,
                            'extended_type': 'EICAR virus test files',
                            'filename': 'eicar_true',
                            'md5': '44d88612fea8a8f36de82e1278abb02f',
                            'mimetype': 'text/plain',
                            'sha1': '3395856ce81f2b7382dee72602f798b642f14140',
                            'sha256': '275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f',
                            'size': 68,
                            'type': 'FILE'}],
              'uri': '912bnadf01295'},
     'event': 'bounty',
     'txhash': '0000000000000000000000000000000b'}
    """
    event: ClassVar[str] = 'bounty'
    schema: ClassVar[PSJSONSchema] = PSJSONSchema({
        'properties': {
            'guid': guid,
            'artifact_type': {
                'type':
                'string',
                'enum': [
                    name.lower()
                    for name, value in ArtifactType.__members__.items()
                ],
                'srckey':
                lambda k, e: ArtifactType.to_string(
                    ArtifactType(e['artifactType']))
            },
            'author': ethereum_address,
            'amount': {
                'type': 'string',
            },
            'uri': {
                'srckey': 'artifactURI'
            },
            'expiration': {
                'srckey': 'expirationBlock',
                'type': 'string',
            },
            'metadata': {
                'type': 'string'
            }
        }
    })

    @classmethod
    def to_message(cls, event) -> WebsocketEventMessage[NewBountyMessageData]:
        return MetadataHandler.fetch(super().to_message(event),
                                     validate=BountyMetadata.validate)
def test_fake_artifact_type_from_uppercase_string():
    # arrange
    # act
    artifact_type = ArtifactType.from_string('fake')
    # assert
    assert artifact_type is None
Esempio n. 26
0
    async def run_task(self, task_index):
        conn = aiohttp.TCPConnector(limit=0, limit_per_host=0)
        timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT)
        async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session:
            redis = await aioredis.create_redis_pool(self.redis_uri)
            while not self.finished:
                try:
                    next_job = await redis.blpop(self.queue, timeout=1)
                    if next_job is None:
                        continue

                    _, job = next_job
                    job = json.loads(job.decode('utf-8'))
                    logger.info(f'Got job on task {task_index}', extra={'extra': job})

                    guid = job['guid']
                    uri = job['uri']

                    polyswarmd_uri = job['polyswarmd_uri']

                    if self.api_key and not polyswarmd_uri.startswith('https://'):
                        raise ApiKeyException()

                    index = job['index']
                    chain = job['chain']
                    metadata = job.get('metadata', None)

                    duration = job['duration']
                    timestamp = job['ts']
                    artifact_type = ArtifactType(int(job['artifact_type']))

                    if timestamp + duration <= time.time() // 1:
                        raise ExpiredException()

                except OSError:
                    logger.exception('Redis connection down')
                    continue
                except aioredis.errors.ReplyError:
                    logger.exception('Redis out of memory')
                    continue
                except KeyError as e:
                    logger.exception(f"Bad message format on task {task_index}: {e}")
                    continue
                except ExpiredException:
                    logger.exception(f'Received expired job {guid} index {index}')
                    continue
                except ApiKeyException:
                    logger.exception("Refusing to send API key over insecure transport")
                    continue
                except (AttributeError, TypeError, ValueError):
                    logger.exception('Invalid job received, ignoring')
                    continue

                headers = {'Authorization': self.api_key} if self.api_key is not None else None
                uri = f'{polyswarmd_uri}/artifacts/{uri}/{index}'
                async with self.download_lock:
                    try:
                        response = await session.get(uri, headers=headers)
                        response.raise_for_status()

                        content = await response.read()
                    except aiohttp.ClientResponseError:
                        logger.exception(f'Error fetching artifact {uri} on task {task_index}')
                        continue
                    except asyncio.TimeoutError:
                        logger.exception(f'Timeout fetching artifact {uri} on task {task_index}')
                        continue

                async with self.scan_lock:
                    result = await self.scanner.scan(guid, artifact_type, content, metadata, chain)

                j = json.dumps({
                    'index': index,
                    'bit': result.bit,
                    'verdict': result.verdict,
                    'confidence': result.confidence,
                    'metadata': result.metadata,
                })

                logger.info(f'Scan results on task {task_index}', extra={'extra': j})

                key = f'{self.queue}_{guid}_{chain}_results'
                try:
                    await redis.rpush(key, j)
                    self.tries = 0
                except OSError:
                    logger.exception('Redis connection down')
                except aioredis.errors.ReplyError:
                    logger.exception('Redis out of memory')
def test_none_artifact_type():
    # arrange
    # act
    artifact_type = ArtifactType.from_string(None)
    # assert
    assert artifact_type is None