def main(log, client_log, redis_addr, queue, backend, tasks, download_limit,
scan_limit, api_key, testing, log_format):
"""Entrypoint for the worker driver
Args:
log (str): Logging level for all app logs
client_log (str): Logging level for all polyswarmclient logs
redis_addr (str): Address of redis
backend (str): Backend implementation to use
queue (str): Name of queue to listen on
tasks (int): Number of simultaneous tasks this worker runs
download_limit (int): Number of simultaneous downloads this worker can handle
scan_limit (int): Number of simultaneous scans this worker can handle
api_key(str): API key to use with polyswarmd
testing (int): Mode to process N bounties then exit (optional)
log_format (str): Format to output logs in. `text` or `json`
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, scanner_class = choose_backend(backend)
scanner = scanner_class()
init_logging(['worker', 'microengine', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
logger.info('Running worker with %s tasks', tasks)
worker = Worker(redis_addr, queue, tasks, download_limit, scan_limit,
api_key, testing, scanner)
worker.run()
def main(log, client_log, redis_addr, queue, backend, tasks, download_limit, scan_limit, api_key, testing, log_format,
scan_time_requirement, daily_rate_limit, hourly_rate_limit, minutely_rate_limit, secondly_rate_limit,
allow_key_over_http):
"""Entrypoint for the worker driver
"""
utils.fast_deprecation()
if tasks != 0:
warnings.warn('Use of --tasks or TASKS is deprecated', DeprecationWarning)
tasks = 0 if download_limit == 0 or scan_limit == 0 else max(download_limit, scan_limit)
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
raise FatalError('Invalid log level', 1)
logger_name, scanner_class = choose_backend(backend)
scanner = scanner_class()
init_logging(['worker', 'microengine', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
logger.info('Running worker with %s tasks', tasks if tasks > 0 else 'unlimited')
worker = Worker(redis_addr, queue, tasks, download_limit, scan_limit, api_key, testing, scanner,
scan_time_requirement, daily_rate_limit, hourly_rate_limit, minutely_rate_limit, secondly_rate_limit,
allow_key_over_http)
worker.run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, chains, watchdog, log_format,
submission_rate):
"""
Entrypoint for the ambassador driver
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, ambassador_class = choose_backend(backend)
init_logging(['ambassador', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
ambassador_class.connect(polyswarmd_addr,
keyfile,
password,
api_key=api_key,
testing=testing,
insecure_transport=insecure_transport,
chains=set(chains),
watchdog=watchdog,
submission_rate=submission_rate).run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport,
allow_key_over_http, chains, watchdog, log_format, submission_rate):
"""
Entrypoint for the ambassador driver
"""
utils.fast_deprecation()
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
raise FatalError('Invalid log level', 1)
logger_name, ambassador_class = choose_backend(backend)
init_logging(['ambassador', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport)
if insecure_transport:
warnings.warn('--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`',
DeprecationWarning)
ambassador_class.connect(polyswarmd_addr, keyfile, password,
api_key=api_key, testing=testing,
chains=set(chains), watchdog=watchdog,
submission_rate=submission_rate).run()
def main(log, client_log, redis_addr, queue, backend, tasks, download_limit,
scan_limit, api_key, testing, log_format):
"""Entrypoint for the worker driver
"""
if tasks != 0:
warnings.warn('Use of --tasks or TASKS is deprecated',
DeprecationWarning)
tasks = 0 if download_limit == 0 or scan_limit == 0 else max(
download_limit, scan_limit)
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, scanner_class = choose_backend(backend)
scanner = scanner_class()
init_logging(['worker', 'microengine', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
logger.info('Running worker with %s tasks',
tasks if tasks > 0 else 'unlimited')
worker = Worker(redis_addr, queue, tasks, download_limit, scan_limit,
api_key, testing, scanner)
worker.run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, chains, log_format, artifact_type):
"""
Entrypoint for the arbiter driver
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, arbiter_class = choose_backend(backend)
init_logging(['arbiter', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
artifact_types = None
if artifact_type:
artifact_types = [
ArtifactType.from_string(artifact) for artifact in artifact_type
]
arbiter_class.connect(polyswarmd_addr,
keyfile,
password,
api_key=api_key,
testing=testing,
insecure_transport=insecure_transport,
chains=set(chains),
artifact_types=artifact_types).run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, allow_key_over_http, chains, log_format,
artifact_type, bid_strategy, accept, exclude, filter, confidence):
""" Entrypoint for the microengine driver
"""
utils.fast_deprecation()
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
raise FatalError('Invalid log level', 1)
polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key,
allow_key_over_http,
insecure_transport)
if insecure_transport:
warnings.warn(
'--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`',
DeprecationWarning)
logger_name, microengine_class = choose_backend(backend)
bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy)
artifact_types = None
init_logging(['microengine', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
if artifact_type:
artifact_types = [
ArtifactType.from_string(artifact) for artifact in artifact_type
]
filter_accept = filter.get('accept', [])
filter_reject = filter.get('reject', [])
if accept or exclude:
warnings.warn(
'Options `--exclude|accept key:value` are deprecated, please switch to'
' `--filter accept|reject key comparison value`',
DeprecationWarning)
filter_accept.extend(accept)
filter_reject.extend(exclude)
favor = confidence.get('favor', [])
penalize = confidence.get('penalize', [])
microengine_class.connect(
polyswarmd_addr,
keyfile,
password,
api_key=api_key,
artifact_types=artifact_types,
bid_strategy=bid_strategy_class(),
bounty_filter=BountyFilter(filter_accept, filter_reject),
chains=set(chains),
confidence_modifier=ConfidenceModifier(favor, penalize),
testing=testing).run()
def cli(log, client_log, log_format):
"""
Entrypoint for the balance manager driver
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
raise FatalError('Invalid log level', 1)
init_logging(['balancemanager'], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, chains, log_format, artifact_type,
bid_strategy, accept, exclude):
"""Entrypoint for the microengine driver
Args:
log (str): Logging level for all app logs
client_log (str): Logging level for all polyswarmclient logs
polyswarmd_addr(str): Address of polyswarmd
keyfile (str): Path to private key file to use to sign transactions
password (str): Password to decrypt the encrypted private key
backend (str): Backend implementation to use
api_key(str): API key to use with polyswarmd
testing (int): Mode to process N bounties then exit (optional)
insecure_transport (bool): Connect to polyswarmd without TLS
chains (list[str]): List of chains on which to scan artifacts
log_format (str): Format to output logs in. `text` or `json`
artifact_type (list[str]): List of artifact types to scan
bid_strategy (str): Bid strategy module name
accept (list[tuple[str]]): List of excluded mimetypes
exclude (list[tuple[str]]): List of excluded mimetypes
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, microengine_class = choose_backend(backend)
bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy)
artifact_types = None
init_logging(['microengine', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
if artifact_type:
artifact_types = [
ArtifactType.from_string(artifact) for artifact in artifact_type
]
microengine_class.connect(polyswarmd_addr,
keyfile,
password,
api_key=api_key,
testing=testing,
insecure_transport=insecure_transport,
chains=set(chains),
artifact_types=artifact_types,
exclude=exclude,
accept=accept,
bid_strategy=bid_strategy_class()).run()
def main(log, log_format, loop_update_threshold, average_bounty_wait_threshold):
if 'liveliness' in sys.argv[0]:
warnings.simplefilter('module', category=DeprecationWarning)
warnings.warn('liveliness is deprecated, use liveness', DeprecationWarning)
loglevel = getattr(logging, log.upper(), None)
if not isinstance(loglevel, int):
logging.error('invalid log level')
raise FatalError('Invalid log level', 1)
init_logging(['liveness'], log_format, loglevel)
liveness_check = LocalLivenessCheck(loop_update_threshold, average_bounty_wait_threshold)
if not liveness_check.check():
raise FatalError('Liveness check failed', 1)
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, chains, log_format, artifact_type,
bid_strategy, accept, exclude, filter, confidence):
"""Entrypoint for the microengine driver
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, microengine_class = choose_backend(backend)
bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy)
artifact_types = None
init_logging(['microengine', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
if artifact_type:
artifact_types = [
ArtifactType.from_string(artifact) for artifact in artifact_type
]
filter_accept = filter.get('accept', [])
filter_reject = filter.get('reject', [])
if accept or exclude:
logger.warning(
'Options `--exclude|accept key:value` are deprecated, please switch to `--filter '
'accept|reject key comparison value`')
filter_accept.extend(accept)
filter_reject.extend(exclude)
favor = confidence.get('favor', [])
penalize = confidence.get('penalize', [])
microengine_class.connect(
polyswarmd_addr,
keyfile,
password,
api_key=api_key,
artifact_types=artifact_types,
bid_strategy=bid_strategy_class(),
bounty_filter=BountyFilter(filter_accept, filter_reject),
chains=set(chains),
confidence_modifier=ConfidenceModifier(favor, penalize),
insecure_transport=insecure_transport,
testing=testing).run()
def cli(log, client_log, log_format):
"""
Entrypoint for the balance manager driver
Args:
log (str): Log level for balancemanger module logs
client_log (str): Log level for all polyswarmclient module logs
log_format (str): Choose either json, or text log format
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
init_logging(['balancemanager'], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, chains, log_format, artifact_type):
"""Entrypoint for the arbiter driver
Args:
log (str): Logging level for all app logs
client_log (str): Logging level for all polyswarmclient logs
polyswarmd_addr(str): Address of polyswarmd
keyfile (str): Path to private key file to use to sign transactions
password (str): Password to decrypt the encrypted private key
backend (str): Backend implementation to use
api_key(str): API key to use with polyswarmd
testing (int): Mode to process N bounties then exit (optional)
insecure_transport (bool): Connect to polyswarmd without TLS
chains (List[str]): Chain(s) to operate on
log_format (str): Format to output logs in. `text` or `json`
artifact_type (list[str]): List of artifact types to scan
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, arbiter_class = choose_backend(backend)
init_logging(['arbiter', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
artifact_types = None
if artifact_type:
artifact_types = [
ArtifactType.from_string(artifact) for artifact in artifact_type
]
arbiter_class.connect(polyswarmd_addr,
keyfile,
password,
api_key=api_key,
testing=testing,
insecure_transport=insecure_transport,
chains=set(chains),
artifact_types=artifact_types).run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, chains, watchdog, log_format,
submission_rate):
"""Entrypoint for the ambassador driver
Args:
log (str): Logging level for all app logs
client_log (str): Logging level for all polyswarmclient logs
polyswarmd_addr(str): Address of polyswarmd
keyfile (str): Path to private key file to use to sign transactions
password (str): Password to decrypt the encrypted private key
backend (str): Backend implementation to use
api_key(str): API key to use with polyswarmd
testing (int): Mode to process N bounties then exit (optional)
insecure_transport (bool): Connect to polyswarmd without TLS
chains (List[str]): Chain(s) to operate on
watchdog (int): Number of blocks to look back and see if bounties are being submitted
log_format (str): Format to output logs in. `text` or `json`
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
sys.exit(-1)
logger_name, ambassador_class = choose_backend(backend)
init_logging(['ambassador', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
ambassador_class.connect(polyswarmd_addr,
keyfile,
password,
api_key=api_key,
testing=testing,
insecure_transport=insecure_transport,
chains=set(chains),
watchdog=watchdog,
submission_rate=submission_rate).run()
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend,
testing, insecure_transport, allow_key_over_http, chains, log_format,
artifact_type):
"""
Entrypoint for the arbiter driver
"""
loglevel = getattr(logging, log.upper(), None)
clientlevel = getattr(logging, client_log.upper(), None)
if not isinstance(loglevel, int) or not isinstance(clientlevel, int):
logging.error('invalid log level')
raise FatalError('Invalid log level', 1)
logger_name, arbiter_class = choose_backend(backend)
init_logging(['arbiter', logger_name], log_format, loglevel)
init_logging(['polyswarmclient'], log_format, clientlevel)
polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key,
allow_key_over_http,
insecure_transport)
if insecure_transport:
warnings.warn(
'--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`',
DeprecationWarning)
artifact_types = None
if artifact_type:
artifact_types = [
ArtifactType.from_string(artifact) for artifact in artifact_type
]
arbiter_class.connect(polyswarmd_addr,
keyfile,
password,
api_key=api_key,
testing=testing,
chains=set(chains),
artifact_types=artifact_types).run()
def main(malicious, benign, output, log_format):
init_logging([], log_format)
generate_db(output, malicious, benign)
