async def _get_async_with_retry(self, *args, **kwargs):
"""
Fetch with retry
"""
retrier = riprova.AsyncRetrier(
backoff=riprova.ExponentialBackOff(factor=.5),
evaluator=self.evaluator,
on_retry=self.on_retry)
return await retrier.run(self._get_async, *args, **kwargs)
class SQLPlugin(StoragePluginBase):
def __init__(self, bot_config):
super().__init__(bot_config)
config = self._storage_config
if DATA_URL_ENTRY not in config:
raise Exception(
'You need to specify a connection URL for the database in your'
'config.py. For example:\n'
'STORAGE_CONFIG={\n'
'"data_url": "postgresql://'
'scott:tiger@localhost/mydatabase/",\n'
'}')
# Hack around the multithreading issue in memory only sqlite.
# This mode is useful for testing.
if config[DATA_URL_ENTRY].startswith('sqlite://'):
from sqlalchemy.pool import StaticPool
self._engine = create_engine(
config[DATA_URL_ENTRY],
connect_args={'check_same_thread': False},
poolclass=StaticPool,
echo=bot_config.BOT_LOG_LEVEL == logging.DEBUG)
else:
self._engine = create_engine(
config[DATA_URL_ENTRY],
echo=bot_config.BOT_LOG_LEVEL == logging.DEBUG)
self._metadata = MetaData()
self._sessionmaker = sessionmaker()
self._sessionmaker.configure(bind=self._engine)
@riprova.retry(backoff=riprova.ExponentialBackOff(factor=0.5))
def open(self, namespace: str) -> StorageBase:
# Create a table with the given namespace
table = Table(namespace, self._metadata,
Column('key', String(767), primary_key=True),
Column('value', String(32768)),
extend_existing=True)
class NewKV(KV):
pass
mapper(NewKV, table, properties={
'_key': table.c.key,
'_value': table.c.value})
# ensure that the table for this namespace exists
self._metadata.create_all(self._engine)
# create an autonomous session for it.
return SQLStorage(self._sessionmaker(), NewKV)
regex = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
return all(regex.match(x) for x in lookup_value.split("."))
# ***Functions to perform the three different kinds of lookups, plus one to force VT to re-scan a URL***
# Riprova error logger
def on_retry(err, next_try):
# print('Error exception: {}'.format(err))
# print('Next try in {}ms'.format(next_try))
pass
# Function to lookup Domains
@riprova.retry(backoff=riprova.ExponentialBackOff(interval=20,
factor=0.5,
max_interval=60,
max_elapsed=900,
multiplier=1),
on_retry=on_retry)
def lookup_domains(domain):
home = cfg.home_path
temp_file = cfg.home_path + temp_files[0]
url = 'https://www.virustotal.com/vtapi/v2/domain/report'
parameters = {'domain': domain, 'apikey': api_key}
domain_lookup_results = []
try:
response = urllib.request.urlopen(
'%s?%s' % (url, urllib.parse.urlencode(parameters))).read()
time.sleep(cfg.request_wait)
host = BITCOIN_RPC_HOST
if BITCOIN_RPC_USER and BITCOIN_RPC_PASSWORD:
host = "%s:%s@%s" % (quote(BITCOIN_RPC_USER), quote(BITCOIN_RPC_PASSWORD), host,)
if BITCOIN_RPC_PORT:
host = "%s:%s" % (host, BITCOIN_RPC_PORT)
service_url = "%s://%s" % (BITCOIN_RPC_SCHEME, host)
return lambda: Proxy(service_url=service_url, timeout=TIMEOUT)
def rpc_client():
return rpc_client_factory()()
@riprova.retry(
timeout=TIMEOUT,
backoff=riprova.ExponentialBackOff(),
on_retry=on_retry,
error_evaluator=error_evaluator,
)
def bitcoinrpc(*args) -> RpcResult:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("RPC call: " + " ".join(str(a) for a in args))
result = rpc_client().call(*args)
logger.debug("Result: %s", result)
return result
def get_block(block_hash: str):
try:
# flake8: noqa
import riprova
@riprova.retry
def task():
"""Retry operation if it fails with constant backoff"""
@riprova.retry(backoff=riprova.ExponentialBackOff(factor=.5))
def task():
"""Retry operation if it fails using exponential backoff"""
@riprova.retry(timeout=10)
def task():
"""Raises a TimeoutError if the retry loop exceeds from 10 seconds"""
def on_retry(err, next_try):
print('Operation error: {}'.format(err))
print('Next try in: {}ms'.format(next_try))
@riprova.retry(on_retry=on_retry)
def task():
"""Subscribe via function callback to every retry attempt"""
def evaluator(response):
# Force retry operation if not a valid response
if response.status >= 400:
raise RuntimeError('invalid response status')
# Otherwise return False, meaning no retry
return False
@riprova.retry(evaluator=evaluator)
# -*- coding: utf-8 -*-
import riprova
# Store number of function calls for error simulation
calls = 0
# Max number of retries attempts
retries = 5
# Register retriable operation with custom evaluator
@riprova.retry(backoff=riprova.ExponentialBackOff(factor=1, retries=retries))
def mul2(x):
global calls
if calls < 4:
calls += 1
raise RuntimeError('simulated call error')
return x * 2
# Run task
result = mul2(2)
print('Result: {}'.format(result))