def bounty_to_dict(bounty): return { 'guid': str(uuid.UUID(int=bounty[0])), 'artifact_type': ArtifactType.to_string(ArtifactType(bounty[1])), 'author': bounty[2], 'amount': str(bounty[3]), 'uri': bounty[4], 'num_artifacts': bounty[5], 'expiration': bounty[6], 'assigned_arbiter': bounty[7], 'quorum_reached': bounty[8], 'quorum_reached_block': bounty[9], 'quorum_mask': safe_int_to_bool_list(bounty[10], bounty[5]), 'metadata': bounty[11] }
async def post_bounty(self, artifact_type, amount, artifact_uri, duration, chain, api_key=None, metadata=None): """Post a bounty to polyswarmd. Args: artifact_type (ArtifactType): The artifact type in this bounty amount (int): The amount to put up as a bounty artifact_uri (str): URI of artifacts duration (int): Number of blocks to accept new assertions chain (str): Which chain to operate on api_key (str): Override default API key metadata (Optional[str]): Optional metadata Returns: Response JSON parsed from polyswarmd containing emitted events """ bounty_fee = await self.parameters[chain].get('bounty_fee') bloom = await self.calculate_bloom(artifact_uri) num_artifacts = await self.__client.get_artifact_count(artifact_uri) transaction = PostBountyTransaction( self.__client, ArtifactType.to_string(artifact_type), amount, bounty_fee, artifact_uri, num_artifacts, duration, bloom, metadata) success, result = await transaction.send(chain, api_key=api_key) if not success or 'bounties' not in result: logger.error('Expected bounty, received', extra={'extra': result}) return result.get('bounties', [])
def collect_metrics(scan: 'ScanResult', start: 'float', artifact_type: 'ArtifactType'): """Collect application metrics from this scan""" # Collect timing information statsd.timing(SCAN_TIME, perf_counter() - start) type_tag = 'type:%s' % ArtifactType.to_string(artifact_type) if scan.bit is True: if scan.verdict is True: verdict_tag = 'verdict:malicious' elif scan.verdict is False: verdict_tag = 'verdict:benign' elif scan.verdict is None: verdict_tag = 'verdict:none' else: verdict_tag = 'verdict:invalid.%s' % type(scan.verdict).__name__ if verbose: statsd.increment(SCAN_VERDICT, tags=[type_tag, verdict_tag]) statsd.increment(SCAN_SUCCESS, tags=[type_tag, verdict_tag]) elif scan.bit is False: try: # Treat any scan result w/ bit=False & 'scan_error' in metadata as an error statsd.increment(SCAN_FAIL, tags=[ type_tag, 'scan_error:%s' % extract_verdict(scan).__dict__['scan_error'] ]) except (AttributeError, KeyError): # otherwise, the engine is just reporting no result statsd.increment(SCAN_NO_RESULT, tags=[type_tag]) else: statsd.increment(SCAN_TYPE_INVALID, tags=[type_tag])
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, chains, log_format, artifact_type): """ Entrypoint for the arbiter driver """ loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') sys.exit(-1) logger_name, arbiter_class = choose_backend(backend) init_logging(['arbiter', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) artifact_types = None if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] arbiter_class.connect(polyswarmd_addr, keyfile, password, api_key=api_key, testing=testing, insecure_transport=insecure_transport, chains=set(chains), artifact_types=artifact_types).run()
def verify(self, transaction): try: decoded = DecodedTransaction.from_transaction(transaction, self.ABI) except ValueError as e: logger.error('Transaction verification failed: %s', str(e)) return False logger.debug('Expected: %s, Actual: %s', self, decoded) guid, artifact_type, amount, artifact_uri, num_artifacts, duration, bloom, metadata = decoded.parameters bloom_value = 0 for b in bloom: bloom_value = bloom_value << 256 | int(b) artifact_type = ArtifactType(int(artifact_type)) artifact_uri = artifact_uri.decode('utf-8') return decoded.value == 0 and \ artifact_type == self.artifact_type and \ artifact_uri == self.artifact_uri and \ num_artifacts == self.num_artifacts and \ duration == self.duration and \ bloom_value == self.bloom and \ amount == self.amount and \ metadata.decode('utf-8') == self.metadata
def test_scanalytics(statsd, engine_info, use_async, scan_result, verbose_metrics, artifact_kind): is_error = isinstance(scan_result, Exception) args = (None, str(uuid4()), artifact_kind, b'content', {}, 'home') type_tag = 'type:%s' % ArtifactType.to_string(artifact_kind) if use_async: @scanalytics(statsd=statsd, engine_info=engine_info, verbose=verbose_metrics) async def scanfn(self, guid, artifact_type, content, metadata, chain): if is_error: raise scan_result return scan_result result = asyncio.run(scanfn(*args)) else: @scanalytics(statsd=statsd, engine_info=engine_info, verbose=verbose_metrics) def scanfn(self, guid, artifact_type, content, metadata, chain): if is_error: raise scan_result return scan_result result = scanfn(*args) statsd.timing.assert_called_once() assert isinstance(result.metadata, str) result_meta = Verdict.parse_raw(result.metadata) assert result_meta.scanner.signatures_version == engine_info.definitions_version assert result_meta.scanner.vendor_version == engine_info.engine_version if is_error: assert result_meta.__dict__['scan_error'] == scan_result.event_name assert result.bit is False statsd.increment.assert_called_once_with( SCAN_FAIL, tags=[type_tag, f'scan_error:{scan_result.event_name}'] ) else: assert result.verdict is scan_result.verdict assert result.bit is scan_result.bit verdict_tag = 'verdict:malicious' if scan_result.verdict else 'verdict:benign' if scan_result.bit is True: statsd.increment.assert_any_call(SCAN_SUCCESS, tags=[type_tag, verdict_tag]) if verbose_metrics: statsd.increment.assert_any_call( SCAN_VERDICT, tags=[type_tag, verdict_tag], ) assert statsd.increment.call_count == 2 else: assert statsd.increment.call_count == 1 elif scan_result.bit is False: if verbose_metrics: statsd.increment.assert_called_once_with(SCAN_NO_RESULT, tags=[type_tag])
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, allow_key_over_http, chains, log_format, artifact_type, bid_strategy, accept, exclude, filter, confidence): """ Entrypoint for the microengine driver """ utils.fast_deprecation() loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) if insecure_transport: warnings.warn( '--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`', DeprecationWarning) logger_name, microengine_class = choose_backend(backend) bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy) artifact_types = None init_logging(['microengine', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] filter_accept = filter.get('accept', []) filter_reject = filter.get('reject', []) if accept or exclude: warnings.warn( 'Options `--exclude|accept key:value` are deprecated, please switch to' ' `--filter accept|reject key comparison value`', DeprecationWarning) filter_accept.extend(accept) filter_reject.extend(exclude) favor = confidence.get('favor', []) penalize = confidence.get('penalize', []) microengine_class.connect( polyswarmd_addr, keyfile, password, api_key=api_key, artifact_types=artifact_types, bid_strategy=bid_strategy_class(), bounty_filter=BountyFilter(filter_accept, filter_reject), chains=set(chains), confidence_modifier=ConfidenceModifier(favor, penalize), testing=testing).run()
def __init__(self, artifact_type, amount, artifact_uri, num_artifacts, duration, bloom, metadata): super().__init__((UNKNOWN_PARAMETER, amount, artifact_uri, num_artifacts, duration, bloom, metadata)) self.artifact_type = ArtifactType.from_string(artifact_type) self.amount = amount self.artifact_uri = artifact_uri self.num_artifacts = num_artifacts self.duration = duration self.bloom = bloom self.metadata = metadata
def make_bounty(): return { 'guid': str(uuid.uuid4()), 'artifact_type': ArtifactType.to_string(ArtifactType.FILE), 'author': random_address(), 'amount': random.randint(0, 10**18), 'uri': random_ipfs_uri(), 'expiration': random.randint(0, 1000), 'metadata': None, }
def post_bounties(): config = app.config['POLYSWARMD'] session = app.config['REQUESTS_SESSION'] account = g.chain.w3.toChecksumAddress(g.eth_address) base_nonce = int( request.args.get('base_nonce', g.chain.w3.eth.getTransactionCount(account))) body = request.get_json() try: _post_bounties_schema(body) except fastjsonschema.JsonSchemaException as e: return failure('Invalid JSON: ' + e.message, 400) guid = uuid.uuid4() artifact_type = ArtifactType.from_string(body['artifact_type']) amount = int(body['amount']) artifact_uri = body['uri'] duration_blocks = body['duration'] metadata = body.get('metadata', '') try: arts = config.artifact.client.ls(artifact_uri, session) except HTTPError as e: return failure(e.response.content, e.response.status_code) except ArtifactException: logger.exception('Failed to ls given artifact uri') return failure(f'Failed to check artifact uri', 500) if amount < eth.bounty_amount_min( g.chain.bounty_registry.contract) * len(arts): return failure('Invalid bounty amount', 400) if metadata and not config.artifact.client.check_uri(metadata): return failure('Invalid bounty metadata URI (should be IPFS hash)', 400) num_artifacts = len(arts) bloom = calculate_bloom(arts) approve_amount = amount + eth.bounty_fee(g.chain.bounty_registry.contract) transactions = [ build_transaction( g.chain.nectar_token.contract.functions.approve( g.chain.bounty_registry.contract.address, approve_amount), base_nonce), build_transaction( g.chain.bounty_registry.contract.functions.postBounty( guid.int, artifact_type.value, amount, artifact_uri, num_artifacts, duration_blocks, bloom, metadata), base_nonce + 1), ] return success({'transactions': transactions})
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, chains, log_format, artifact_type, bid_strategy, accept, exclude): """Entrypoint for the microengine driver Args: log (str): Logging level for all app logs client_log (str): Logging level for all polyswarmclient logs polyswarmd_addr(str): Address of polyswarmd keyfile (str): Path to private key file to use to sign transactions password (str): Password to decrypt the encrypted private key backend (str): Backend implementation to use api_key(str): API key to use with polyswarmd testing (int): Mode to process N bounties then exit (optional) insecure_transport (bool): Connect to polyswarmd without TLS chains (list[str]): List of chains on which to scan artifacts log_format (str): Format to output logs in. `text` or `json` artifact_type (list[str]): List of artifact types to scan bid_strategy (str): Bid strategy module name accept (list[tuple[str]]): List of excluded mimetypes exclude (list[tuple[str]]): List of excluded mimetypes """ loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') sys.exit(-1) logger_name, microengine_class = choose_backend(backend) bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy) artifact_types = None init_logging(['microengine', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] microengine_class.connect(polyswarmd_addr, keyfile, password, api_key=api_key, testing=testing, insecure_transport=insecure_transport, chains=set(chains), artifact_types=artifact_types, exclude=exclude, accept=accept, bid_strategy=bid_strategy_class()).run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, chains, log_format, artifact_type, bid_strategy, accept, exclude, filter, confidence): """Entrypoint for the microengine driver """ loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') sys.exit(-1) logger_name, microengine_class = choose_backend(backend) bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy) artifact_types = None init_logging(['microengine', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] filter_accept = filter.get('accept', []) filter_reject = filter.get('reject', []) if accept or exclude: logger.warning( 'Options `--exclude|accept key:value` are deprecated, please switch to `--filter ' 'accept|reject key comparison value`') filter_accept.extend(accept) filter_reject.extend(exclude) favor = confidence.get('favor', []) penalize = confidence.get('penalize', []) microengine_class.connect( polyswarmd_addr, keyfile, password, api_key=api_key, artifact_types=artifact_types, bid_strategy=bid_strategy_class(), bounty_filter=BountyFilter(filter_accept, filter_reject), chains=set(chains), confidence_modifier=ConfidenceModifier(favor, penalize), insecure_transport=insecure_transport, testing=testing).run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, chains, log_format, artifact_type): """Entrypoint for the arbiter driver Args: log (str): Logging level for all app logs client_log (str): Logging level for all polyswarmclient logs polyswarmd_addr(str): Address of polyswarmd keyfile (str): Path to private key file to use to sign transactions password (str): Password to decrypt the encrypted private key backend (str): Backend implementation to use api_key(str): API key to use with polyswarmd testing (int): Mode to process N bounties then exit (optional) insecure_transport (bool): Connect to polyswarmd without TLS chains (List[str]): Chain(s) to operate on log_format (str): Format to output logs in. `text` or `json` artifact_type (list[str]): List of artifact types to scan """ loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') sys.exit(-1) logger_name, arbiter_class = choose_backend(backend) init_logging(['arbiter', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) artifact_types = None if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] arbiter_class.connect(polyswarmd_addr, keyfile, password, api_key=api_key, testing=testing, insecure_transport=insecure_transport, chains=set(chains), artifact_types=artifact_types).run()
async def run(self, guid, artifact_type, author, amount, uri, expiration, metadata, block_number, txhash, chain): """Run the registered callbacks Args: guid (str): Bounty GUID artifact_type (str): String representation of artifact type author (str): Author of the bounty amount (str): Bounty reward amount uri (str): URI of the artifacts in the bounty expiration (int): Block number the bounty expires on metadata (dict): Dictionary or string of metadata block_number (int): Block number the bounty was posted on txhash (str): Transaction hash which caused the event chain (str): Chain event received on """ return await super().run(guid, ArtifactType.from_string(artifact_type), author, amount, uri, expiration, metadata, block_number, txhash, chain)
async def handle_new_bounty(guid, artifact_type, author, amount, uri, expiration, metadata, block_number, txhash, chain): new_bounty = { 'guid': guid, 'artifact_type': ArtifactType.to_string(artifact_type), 'author': author, 'amount': amount, 'uri': uri, 'expiration': expiration, 'metadata': metadata, } if chain == 'home': assert new_bounty == home_bounty home_done.set() elif chain == 'side': assert new_bounty == side_bounty side_done.set() else: raise ValueError('Invalid chain')
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, allow_key_over_http, chains, log_format, artifact_type): """ Entrypoint for the arbiter driver """ utils.fast_deprecation() loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) logger_name, arbiter_class = choose_backend(backend) init_logging(['arbiter', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) if insecure_transport: warnings.warn( '--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`', DeprecationWarning) artifact_types = None if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] arbiter_class.connect(polyswarmd_addr, keyfile, password, api_key=api_key, testing=testing, chains=set(chains), artifact_types=artifact_types).run()
def __post_init__(self): if not BountyMetadata.validate(self.metadata): raise ValueError if not ArtifactType(self.artifact_type): raise ValueError
def test_url_artifact_type_from_uppercase_string(): # arrange # act artifact_type = ArtifactType.from_string('URL') # assert assert artifact_type == ArtifactType.URL
def test_file_artifact_type_from_lowercase_string(): # arrange # act artifact_type = ArtifactType.from_string('file') # assert assert artifact_type == ArtifactType.FILE
def test_file_artifact_type_to_string(): # arrange # act # assert assert ArtifactType.to_string(ArtifactType.FILE) == 'file'
def test_url_artifact_type_to_string(): # arrange # act # assert assert ArtifactType.to_string(ArtifactType.URL) == 'url'
def test_url_artifact_type_from_int(): # arrange # act artifact_type = ArtifactType(1) # assert assert artifact_type == ArtifactType.URL
def test_file_artifact_type_from_int(): # arrange # act artifact_type = ArtifactType(0) # assert assert artifact_type == ArtifactType.FILE
class NewBounty(WebsocketFilterMessage[NewBountyMessageData]): """NewBounty doctest: When the doctest runs, MetadataHandler._substitute_metadata is already defined outside this doctest (in __main__.py). Running this as a doctest will not trigger network IO. >>> event = mkevent({ ... 'guid': 1066, ... 'artifactType': 1, ... 'author': addr1, ... 'amount': 10, ... 'artifactURI': '912bnadf01295', ... 'expirationBlock': 118, ... 'metadata': 'ZWassertionuri'}) >>> decoded_msg(NewBounty.serialize_message(event)) {'block_number': 117, 'data': {'amount': '10', 'artifact_type': 'url', 'author': '0x00000000000000000000000000000001', 'expiration': '118', 'guid': '00000000-0000-0000-0000-00000000042a', 'metadata': [{'bounty_id': 69540800813340, 'extended_type': 'EICAR virus test files', 'filename': 'eicar_true', 'md5': '44d88612fea8a8f36de82e1278abb02f', 'mimetype': 'text/plain', 'sha1': '3395856ce81f2b7382dee72602f798b642f14140', 'sha256': '275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f', 'size': 68, 'type': 'FILE'}], 'uri': '912bnadf01295'}, 'event': 'bounty', 'txhash': '0000000000000000000000000000000b'} """ event: ClassVar[str] = 'bounty' schema: ClassVar[PSJSONSchema] = PSJSONSchema({ 'properties': { 'guid': guid, 'artifact_type': { 'type': 'string', 'enum': [ name.lower() for name, value in ArtifactType.__members__.items() ], 'srckey': lambda k, e: ArtifactType.to_string( ArtifactType(e['artifactType'])) }, 'author': ethereum_address, 'amount': { 'type': 'string', }, 'uri': { 'srckey': 'artifactURI' }, 'expiration': { 'srckey': 'expirationBlock', 'type': 'string', }, 'metadata': { 'type': 'string' } } }) @classmethod def to_message(cls, event) -> WebsocketEventMessage[NewBountyMessageData]: return MetadataHandler.fetch(super().to_message(event), validate=BountyMetadata.validate)
def test_fake_artifact_type_from_uppercase_string(): # arrange # act artifact_type = ArtifactType.from_string('fake') # assert assert artifact_type is None
async def run_task(self, task_index): conn = aiohttp.TCPConnector(limit=0, limit_per_host=0) timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT) async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: redis = await aioredis.create_redis_pool(self.redis_uri) while not self.finished: try: next_job = await redis.blpop(self.queue, timeout=1) if next_job is None: continue _, job = next_job job = json.loads(job.decode('utf-8')) logger.info(f'Got job on task {task_index}', extra={'extra': job}) guid = job['guid'] uri = job['uri'] polyswarmd_uri = job['polyswarmd_uri'] if self.api_key and not polyswarmd_uri.startswith('https://'): raise ApiKeyException() index = job['index'] chain = job['chain'] metadata = job.get('metadata', None) duration = job['duration'] timestamp = job['ts'] artifact_type = ArtifactType(int(job['artifact_type'])) if timestamp + duration <= time.time() // 1: raise ExpiredException() except OSError: logger.exception('Redis connection down') continue except aioredis.errors.ReplyError: logger.exception('Redis out of memory') continue except KeyError as e: logger.exception(f"Bad message format on task {task_index}: {e}") continue except ExpiredException: logger.exception(f'Received expired job {guid} index {index}') continue except ApiKeyException: logger.exception("Refusing to send API key over insecure transport") continue except (AttributeError, TypeError, ValueError): logger.exception('Invalid job received, ignoring') continue headers = {'Authorization': self.api_key} if self.api_key is not None else None uri = f'{polyswarmd_uri}/artifacts/{uri}/{index}' async with self.download_lock: try: response = await session.get(uri, headers=headers) response.raise_for_status() content = await response.read() except aiohttp.ClientResponseError: logger.exception(f'Error fetching artifact {uri} on task {task_index}') continue except asyncio.TimeoutError: logger.exception(f'Timeout fetching artifact {uri} on task {task_index}') continue async with self.scan_lock: result = await self.scanner.scan(guid, artifact_type, content, metadata, chain) j = json.dumps({ 'index': index, 'bit': result.bit, 'verdict': result.verdict, 'confidence': result.confidence, 'metadata': result.metadata, }) logger.info(f'Scan results on task {task_index}', extra={'extra': j}) key = f'{self.queue}_{guid}_{chain}_results' try: await redis.rpush(key, j) self.tries = 0 except OSError: logger.exception('Redis connection down') except aioredis.errors.ReplyError: logger.exception('Redis out of memory')
def test_none_artifact_type(): # arrange # act artifact_type = ArtifactType.from_string(None) # assert assert artifact_type is None