class HackerNewsStore:
@tenacity.retry(stop=tenacity.stop_after_attempt(5),
wait=tenacity.wait_random(min=1, max=2),
before_sleep=tenacity.before_sleep_log(
logger, logging.DEBUG)) # noqa: E501
def add_article(self, article):
connection = DatabaseHelper.get_connection()
with connection:
cursor = connection.cursor()
try:
cursor.execute(self._get_query('add_article'),
(article.id, article.link, article.title))
except psycopg2.IntegrityError as e:
logger.info(e)
def purge(self):
connection = DatabaseHelper.get_connection()
with connection:
cursor = connection.cursor()
lastCreateTime = datetime.date.today() - datetime.timedelta(
days=10) # noqa: E501
cursor.execute(self._get_query('purge_removed_articles'),
(lastCreateTime, ))
logger.info(f"deleted rows:{cursor.rowcount}")
def _get_query(self, query_id):
query = None
if DATABASE_CONFIG['type'] == 'postgre':
query = PostgreQueries[query_id]
else:
raise RuntimeError(
"The specified db type:{} is not supported".format(
DATABASE_CONFIG['type'])) # noqa: E501
return query
def test_wait_double_sum(self):
r = Retrying(wait=tenacity.wait_random(0, 3) + tenacity.wait_fixed(5))
# Test it a few time since it's random
for i in six.moves.range(1000):
w = r.wait(1, 5)
self.assertLess(w, 8)
self.assertGreaterEqual(w, 5)
def __init__(self, config: dict, logger, exchange):
self.conf = config
self.logger = logger
self._exchange = exchange
cfg = config.bot
self._purchase_size_percentage = cfg["purchase_size_percentage"]
self._update_interval = cfg["update_interval"]
self._symbol = cfg["symbol"]
self.util = Utils(self._exchange, self.logger)
self._aretry = tenacity.AsyncRetrying(
wait=tenacity.wait_random(0, 2),
retry=(tenacity.retry_if_exception(ccxt.DDoSProtection)
| tenacity.retry_if_exception(ccxt.RequestTimeout)))
self._markets = {}
if self._symbol:
self._markets[self._symbol] = {
"curr_sold": 0,
"curr_bought": 0,
"buys": 0,
"sells": 0
}
def save_configuration(self, net_connect):
retry_kwargs = {
'wait': tenacity.wait_random(min=2, max=6),
'reraise': False,
'stop': tenacity.stop_after_delay(30)
}
@tenacity.retry(**retry_kwargs)
def _save():
try:
net_connect.write(("copy running-config startup-config" +
'\n').encode('utf-8'))
error_msg = net_connect.read_until("Invalid", timeout=2)
logger.debug("execute command :%s" % error_msg)
if "Configuration update aborted" in error_msg:
result = net_connect.read_very_eager()
logger.debug("execute command failed.error msg:%s" %
result)
raise exceptions.ConfigSwitchError(
command="copy running-config startup-config",
error=result)
except Exception:
raise
return error_msg
return _save()
class GitUrlWatcher(SubTask):
def __init__(self, app_config: Dict):
super().__init__(name="git repo watcher")
self.watched_repos = []
watched_compose_files_config = app_config["main"]["watched_git_repositories"]
for config in watched_compose_files_config:
repo = GitRepo(
repo_id=config["id"],
repo_url=config["url"],
branch=config["branch"],
tags=config["tags"],
pull_only_files=config["pull_only_files"],
username=config["username"],
password=config["password"],
paths=config["paths"],
)
self.watched_repos.append(repo)
async def init(self) -> Dict:
description = await _init_repositories(self.watched_repos)
return description
@retry(
reraise=True,
stop=stop_after_attempt(NUMBER_OF_ATTEMPS),
wait=wait_random(min=1, max=MAX_TIME_TO_WAIT_S),
after=after_log(log, logging.DEBUG),
)
async def check_for_changes(self) -> Dict:
return await _check_repositories(self.watched_repos)
async def cleanup(self):
await _delete_repositories(self.watched_repos)
def __init__(self, logger=None, config=None, db=None):
self._logger = logger
if config:
self._rsi_timeframe = config["rsi_timeframe"]
self._interval = config["update_interval"]
self._over_bought = config["over_bought"]
self._rsi_period = config["rsi_period"]
self._free_fall = config["free_fall"]
self._over_sold = config["over_sold"]
self._mooning = config["mooning"]
self._db = db
self._exchange_market_prices = {}
self._significant_markets = set()
self._aretry = tenacity.AsyncRetrying(
wait=tenacity.wait_random(0, 3),
retry=(
tenacity.retry_if_exception(ccxt.DDoSProtection) |
tenacity.retry_if_exception(ccxt.RequestTimeout) |
tenacity.retry_if_exception(aiohttp.ServerDisconnectedError)
)
)
def _blocking_poll(self, timeout=None):
"""Poll and wait for the Future to be resolved.
Args:
timeout (int): How long to wait for the operation to complete.
If None, wait indefinitely.
"""
if self._result_set:
return
retry_on = tenacity.retry_if_result(
functools.partial(operator.is_not, True))
# Use exponential backoff with jitter.
wait_on = (tenacity.wait_exponential(multiplier=1, max=10) +
tenacity.wait_random(0, 1))
if timeout is None:
retry = tenacity.retry(retry=retry_on, wait=wait_on)
else:
retry = tenacity.retry(retry=retry_on,
wait=wait_on,
stop=tenacity.stop_after_delay(timeout))
try:
retry(self.done)()
except tenacity.RetryError as exc:
six.raise_from(
concurrent.futures.TimeoutError(
'Operation did not complete within the designated '
'timeout.'), exc)
def retry_on_conflict(func):
wrapper = tenacity.retry(stop=tenacity.stop_after_attempt(11),
wait=tenacity.wait_random(max=0.002),
retry=tenacity.retry_if_exception_type(
exception.ConcurrentTransaction),
reraise=True)
return wrapper(func)
async def get_current_version(self) -> version.Version:
async with httpx.AsyncClient(timeout=self.timeout) as client:
try:
async for attempt in AsyncRetrying(
stop=stop_after_attempt(5),
reraise=True,
retry=retry_if_exception_type(httpx.ConnectTimeout),
wait=wait_random(min=self.timeout,
max=self.timeout * 10),
):
with attempt:
r = await client.get(self.checkurl)
except (httpx.HTTPStatusError, httpx.ReadTimeout):
return None
webpage = r.text
# FIXME: this is a hardcode
tarball_regex = re.compile(r"\bemacs-[0-9.]+\.tar\.[a-z]*\b")
version_regex = re.compile(r"\b[0-9.]+\b")
currentversion = None
for tarball in tarball_regex.findall(webpage):
versionstring = version_regex.findall(tarball)[0]
if versionstring.endswith("."):
versionstring = versionstring[:-1]
emacsversion = version.parse(versionstring)
if currentversion is None or currentversion < emacsversion:
currentversion = emacsversion
return emacsversion
def _retry_send(self, function: Callable, attempt=10, *args, **kwargs):
retry_configuration = tenacity.Retrying(
stop=tenacity.stop_after_attempt(attempt),
wait=tenacity.wait_fixed(3) + tenacity.wait_random(0, 2),
after=tenacity.after_log(logger, logger.level) if logger else None,
reraise=True,
)
return retry_configuration(function, *args, **kwargs)
def retry_on_conflict(func):
wrapper = tenacity.retry(
stop=tenacity.stop_after_attempt(11),
wait=tenacity.wait_random(max=0.002),
retry=tenacity.retry_if_exception_type(exception.ConcurrentTransaction),
reraise=True,
)
return wrapper(func)
def wrapped_retry(*args, **kwargs):
retryer = tenacity.Retrying(
wait=tenacity.wait_random(1, 3),
retry=tenacity.retry_if_exception_type(exception_type),
stop=tenacity.stop_after_attempt(retries + 1),
reraise=True,
before_sleep=log_retry)
return retryer(func, *args, **kwargs)
def __init__(self,
min=0,
multiplier=1,
max=tenacity._utils.MAX_WAIT,
exp_base=2):
super(wait_random_exponential, self).__init__(multiplier=multiplier,
max=(max - min),
exp_base=exp_base)
self._random = tenacity.wait_random(min=min, max=(min + multiplier))
def __init__(self, exchange: ccxt.Exchange, logger):
self._exchange = exchange
self._logger = logger
self._aretry = tenacity.AsyncRetrying(
wait=tenacity.wait_random(0, 2),
retry=(tenacity.retry_if_exception(ccxt.DDoSProtection)
| tenacity.retry_if_exception(ccxt.RequestTimeout)))
self.purchase_strategies = {}
def retry(cls, timeout: float, callback, throw: Exception, *args, **kwargs) -> any:
@tenacity.retry(wait=tenacity.wait_random(min=timeout/1000, max=timeout/100),
stop=tenacity.stop_after_delay(timeout),
reraise=True)
def retry_impl(callback, *args, **kwargs):
result = callback(*args, **kwargs)
if result:
return result
raise throw('Operation timed out')
return retry_impl(callback, *args, **kwargs)
def test_wait_arbitrary_sum(self):
r = Retrying(wait=sum([tenacity.wait_fixed(1),
tenacity.wait_random(0, 3),
tenacity.wait_fixed(5),
tenacity.wait_none()]))
# Test it a few time since it's random
for i in six.moves.range(1000):
w = r.wait(1, 5)
self.assertLess(w, 9)
self.assertGreaterEqual(w, 6)
def __init__(self):
# throttling
self.throttling_wait = wait_chain(
# always wait 20-40s first
wait_fixed(20) + wait_random(0, 20),
# wait 20-40s again
wait_fixed(20) + wait_random(0, 20),
# wait from 30 to 630s, with full jitter and exponentially
# increasing max wait time
wait_fixed(30) + wait_random_exponential(multiplier=1, max=600)
)
# connection errors, other client and server failures
self.network_wait = (
# wait from 3s to ~1m
wait_random(3, 7) + wait_random_exponential(multiplier=1, max=55)
)
self.server_wait = self.network_wait
def retry_blob_ingest_to_adx(container_name: str, blob_file_path: str,
new_container_name: str, new_blob_file_path: str) -> None:
""" Re-trigger the ingest pipeline by moving blob to retry folder """
# Add a random retry delay plus exponential backoff to mitigate the concurrent access to Azure
retryer = Retrying(stop=stop_after_attempt(BLOB_REQ_MAX_ATTEMPT),
wait=wait_random(0, 5) + wait_exponential(multiplier=1, min=2,
max=BLOB_REQ_MAX_RETRY_DELAY_SEC),
reraise=True)
retryer(move_blob_file, AZURE_STORAGE_CONNECTION_STRING, container_name, new_container_name,
blob_file_path, new_blob_file_path)
def test_random_sleep(self):
r = Retrying(wait=tenacity.wait_random(min=1, max=20))
times = set()
for x in six.moves.range(1000):
times.add(r.wait(1, 6546))
# this is kind of non-deterministic...
self.assertTrue(len(times) > 1)
for t in times:
self.assertTrue(t >= 1)
self.assertTrue(t < 20)
class BasicParticipant():
""" Base class for an FFL general user """
def __init__(self, context: Context):
"""
Class initializer.
Throws: An exception on failure
:param context: connection details
:type context: :class:`.Context`
"""
if not context:
raise Exception('Credentials must be specified.')
self.messenger = None
self.context = context
self.queue = None
def __enter__(self):
"""
Context manager enters - call connect.
Throws: An exception on failure
:return: self
:rtype: :class:`.BasicParticipant`
"""
return self.connect()
def __exit__(self, *args):
"""
Context manager exits - call close
Throws: An exception on failure
"""
self.close()
@tenacity.retry(wait=tenacity.wait_random(min=1, max=2),
stop=tenacity.stop_after_delay(5),
reraise=True)
def connect(self):
"""
Connect to the messaging system.
Throws: An exception on failure
:return: self
:rtype: :class:`.BasicParticipant`
"""
self.messenger = Messenger(self.context, subscribe_queue=self.queue)
return self
def close(self) -> None:
"""
Close the connection to the messaging system.
Throws: An exception on failure
"""
self.messenger.stop()
self.messenger = None
def enviroplus_retry(fn: Coroutine) -> Coroutine:
@wraps(fn)
@ten.retry(retry=ten.retry_if_exception_type(
(aiohttp.ClientError, aiohttp.http_exceptions.HttpProcessingError)),
reraise=True,
wait=ten.wait_random(1, 3),
stop=ten.stop_after_attempt(3),
before_sleep=ten.before_sleep_log(logger, logging.DEBUG))
async def wrapped_fn(*args, **kwargs):
return await fn(*args, **kwargs)
return wrapped_fn
def bind(self, req, body, instance_id, app_id):
db_service = db_cf.get_service_for_instance(instance_id)
if not db_service:
return {}
service_id = db_service.service_id
environment_id = db_service.environment_id
token = req.headers['X-Auth-Token']
m_cli = _get_muranoclient(token, req)
session_id = create_session(m_cli, environment_id)
env = m_cli.environments.get(environment_id, session_id)
LOG.debug('Got environment {0}'.format(env))
service = self._get_service(env, service_id)
LOG.debug('Got service {0}'.format(service))
# NOTE(starodubcevna): Here we need to find an action which will return
# us needed credentials. By default we will look for getCredentials
# action.
result = {}
try:
actions = service['?']['_actions']
for action_id in list(actions):
if 'getCredentials' in action_id:
@tenacity.retry(
retry=tenacity.retry_if_exception_type(TypeError),
wait=tenacity.wait_random(min=1, max=10),
stop=tenacity.stop_after_delay(30),
reraise=True)
def _get_creds(client, task_id, environment_id):
result = m_cli.actions.get_result(
environment_id, task_id)['result']
return result
task_id = m_cli.actions.call(environment_id, action_id)
result = _get_creds(m_cli, task_id, environment_id)
if not result:
LOG.warning(
_LW("This application doesn't have action "
"getCredentials"))
return response.Response(status=500)
except KeyError:
# NOTE(starodubcevna): In CF service broker API spec return
# code for failed bind is not present, so we will return 500.
LOG.warning(_LW("This application doesn't have actions at all"))
return response.Response(status=500)
if 'credentials' in list(result):
return result
else:
return {'credentials': result}
class PolicyInsights2:
def __init__(self, credentials=None):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
self.credentials = AzureConnections().get_authenticated_client()
@retry(wait=wait_random(min=1, max=3), stop=stop_after_attempt(4))
def policy_states_summarize_for_subscription(self, subscriptionId):
"""
This method calls the policy compliance using the policy insights and query policy state
:rtype: response
:param subscriptionId:
:return:
"""
api_endpoint = self.config["AZURESDK"][
'policy_states_summarize_for_subscription']
api_endpoint = api_endpoint.format(subscriptionId=subscriptionId)
with request_authenticated_azure_session() as req:
policy_states_summary = req.post(api_endpoint)
return policy_states_summary
@retry(wait=wait_random(min=1, max=3), stop=stop_after_attempt(4))
def policy_states_summarize_for_policy_definition(self, subscription_id,
policy_definition_name):
api_endpoint = self.config["AZURESDK"][
'policy_states_summarize_for_policy_definition']
api_endpoint = api_endpoint.format(
subscriptionId=subscription_id,
policyDefinitionName=policy_definition_name)
with request_authenticated_azure_session() as req:
policy_states_summary = req.post(api_endpoint)
return policy_states_summary
def test_random_sleep_without_min(self):
r = Retrying(wait=tenacity.wait_random(max=2))
times = set()
times.add(r.wait(1, 6546))
times.add(r.wait(1, 6546))
times.add(r.wait(1, 6546))
times.add(r.wait(1, 6546))
# this is kind of non-deterministic...
self.assertTrue(len(times) > 1)
for t in times:
self.assertTrue(t >= 0)
self.assertTrue(t <= 2)
class DockerRegistriesWatcher(SubTask):
def __init__(self, app_config: Dict, stack_cfg: Dict):
super().__init__(name="dockerhub repo watcher")
# get all the private registries
self.private_registries = app_config["main"]["docker_private_registries"]
# get all the images to check for
self.watched_repos = []
if "services" in stack_cfg:
for service_name in stack_cfg["services"].keys():
if "image" in stack_cfg["services"][service_name]:
image_url = stack_cfg["services"][service_name]["image"]
self.watched_repos.append({
"image": image_url
})
async def init(self):
log.debug("initialising docker watcher..")
with docker_client(self.private_registries) as client:
for repo in self.watched_repos:
try:
registry_data = client.images.get_registry_data(repo["image"])
log.debug("accessed to image %s: %s", repo["image"], registry_data.attrs)
repo["registry_data_attrs"] = registry_data.attrs
except docker.errors.APIError:
# in case a new service that is not yet in the registry was added
log.warning("could not find image %s, maybe a new image was added to the stack??", repo["image"])
repo["registry_data_attrs"] = ""
log.debug("docker watcher initialised")
@retry(reraise=True, stop=stop_after_attempt(NUMBER_OF_ATTEMPS), wait=wait_random(min=1, max=MAX_TIME_TO_WAIT_S), after=after_log(log, logging.DEBUG))
async def check_for_changes(self) -> Dict:
changes = {}
with docker_client(self.private_registries) as client:
for repo in self.watched_repos:
try:
registry_data = client.images.get_registry_data(repo["image"])
if repo["registry_data_attrs"]['Descriptor'] != registry_data.attrs['Descriptor']:
log.info("docker image %s signature changed from %s to %s!", repo["image"], repo["registry_data_attrs"], registry_data.attrs)
changes[repo['image']] = "image signature changed"
except docker.errors.APIError:
if repo["registry_data_attrs"]:
# in that case something is wrong...either docker or config
log.exception("Error while retrieving image %s in registry", repo["image"])
# raise
else:
# in that case the registry does not contain yet the new service
log.warning("the image %s is still not available in the registry", repo["image"])
return changes
async def cleanup(self):
pass
async def fetch_url(url: str, timeout: int = 5, retry: int = 5):
async with httpx.AsyncClient(timeout=timeout) as client:
try:
async for attempt in AsyncRetrying(
stop=stop_after_attempt(5),
reraise=True,
retry=retry_if_exception_type(httpx.ConnectTimeout),
wait=wait_random(min=timeout, max=timeout * 10),
):
with attempt:
r = await client.get(url)
except (httpx.HTTPStatusError, httpx.ReadTimeout) as e:
return str(e)
return r.json()
class Comms_worker:
"""
"""
def __init__(self, commsffl, worker_real_name='Anonymous'):
"""
"""
self.id = worker_real_name # unused by now...
#self.task_name = task_name
#self.commsffl = ffl.Factory.participant(context_w, task_name=self.task_name)
self.name = 'pycloudmessenger'
self.commsffl = commsffl
def send(self, message, address=None):
try:
# address is not used here, a worker can only send to the master
with self.commsffl:
self.commsffl.send(message)
except Exception as err:
print('\n')
print('*' * 80)
print('Pycloudmessenger ERROR at send: %s' % err)
print('*' * 80)
print('\n')
raise
def receive(self, timeout=1):
try:
with self.commsffl:
packet = self.commsffl.receive(timeout)
message = packet.content
except Exception as err:
if 'pycloudmessenger.ffl.fflapi.TimedOutException' not in str(
type(err)): # we skip the normal timeouts
print('\n')
print('*' * 80)
print('Pycloudmessenger ERROR at receive: %s' % err)
print('*' * 80)
print('\n')
else:
message = None
raise
return message
@tenacity.retry(stop=tenacity.stop_after_attempt(5),
wait=tenacity.wait_random(min=1, max=3))
def receive_poms_123(self, timeout=10):
with self.commsffl:
packet = self.commsffl.receive(timeout)
return packet
class PoloniexBaseAPI(metaclass=ABCMeta):
@abstractmethod
def __init__(
self,
*,
requests_per_second: int = REQUESTS_PER_SECOND,
) -> None:
self._rate_limiter = RateLimiter(requests_per_second, 1, 1)
self._requests_per_second = requests_per_second
self._session = Session()
@tenacity.retry(
retry=tenacity.retry_if_exception(_retry_poloniex_error),
wait=tenacity.wait_exponential() + tenacity.wait_random(0, 1),
stop=tenacity.stop_after_attempt(4) | tenacity.stop_after_delay(8),
reraise=True,
)
def request(
self,
*args,
**kwargs,
) -> Optional[Union[Dict[str, Any], List[Dict[str, Any]]]]:
cls_name = type(self).__name__
while self._rate_limiter.check(cls_name) is False:
sleep(1 / self._requests_per_second)
request = Request(*args, **kwargs)
prepared = self._session.prepare_request(request)
response = self._session.send(prepared)
status = response.status_code
if status >= 200 and status < 300:
data = None
try:
data = response.json()
except ValueError:
logger.exception('Error decoding response JSON')
return data
elif status >= 400 and status < 500:
raise PoloniexRequestError(response)
elif status >= 500:
raise PoloniexServerError(response)
else:
# We shouldn't ever get 1xx responses (Poloniex doesn't send them)
# or 3xx responses (requests follows redirects); return None to
# make mypy happy
return None
def save_configuration(self, net_connect, cmd='copy running-config startup-config',):
retry_kwargs = {'wait': tenacity.wait_random(min=2, max=6),
'reraise': False,
'stop': tenacity.stop_after_delay(30)}
@tenacity.retry(**retry_kwargs)
def _save():
try:
output = super(CiscoNxosSSH, net_connect).save_config()
self._check_output(output, cmd)
except Exception:
raise
return output
return _save()
class HttpFetcher:
SSL_VERIFY = os.environ.get("SSL_VERIFY", True) not in FALSY
def __init__(self, logger):
self.logger = logger
if not self.SSL_VERIFY:
logger.warning(
"You have set ssl certificates to not be verified. "
"This may leave you vulnerable. "
"http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification"
)
def get_size(self, url):
try:
head = requests.head(url,
allow_redirects=True,
verify=self.SSL_VERIFY)
return int(head.headers["Content-Length"])
except:
head = requests.get(
url,
allow_redirects=True,
verify=self.SSL_VERIFY,
headers={"Range": "bytes=0-1"},
)
crange = head.headers["Content-Range"]
match = re.search(r"/(\d+)$", crange)
if match:
return int(match.group(1))
self.logger.error(traceback.format_exc())
raise FuseOSError(ENOENT)
@retry(wait=wait_fixed(1) + wait_random(0, 2), stop=stop_after_attempt(2))
def get_data(self, url, start, end):
headers = {
"Range": "bytes={}-{}".format(start, end),
"Accept-Encoding": ""
}
self.logger.info("getting %s %s %s", url, start, end)
r = requests.get(url, headers=headers)
self.logger.info("got %s", r.status_code)
r.raise_for_status()
block_data = np.frombuffer(r.content, dtype=np.uint8)
return block_data
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.readthedocs.org/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
@tenacity.retry(
stop=tenacity.stop_after_attempt(100),
wait=tenacity.wait_random(min=2, max=5),
before=tenacity.before_log(logging.getLogger('tenacity.retry'),
logging.DEBUG),
before_sleep=tenacity.before_sleep_log(
logging.getLogger('tenacity.retry'), logging.INFO),
after=tenacity.after_log(logging.getLogger('tenacity.retry'),
logging.DEBUG))
def try_connect(db):
return db.connect()
with try_connect(engine) as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
with context.begin_transaction():
context.run_migrations()
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.readthedocs.org/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = tenacity.Retrying(
stop=tenacity.stop_after_attempt(100),
wait=tenacity.wait_random(min=2, max=5),
before=tenacity.before_log(logging.getLogger("tenacity.retry"), logging.DEBUG),
before_sleep=tenacity.before_sleep_log(logging.getLogger("tenacity.retry"), logging.INFO),
after=tenacity.after_log(logging.getLogger("tenacity.retry"), logging.DEBUG)
).call(engine.connect)
context.configure(connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
import random
import tenacity
def do_something():
if random.randint(0, 1) == 0:
print('Failure')
raise RuntimeError
print('Success')
@tenacity.retry(wait=tenacity.wait_fixed(10) + tenacity.wait_random(0, 3))
def do_something_and_retry():
do_something()
do_something_and_retry()
def process_data(self):
stations = {}
try:
self.log.info('Processing FFVL data...')
result = requests.get(
'http://data.ffvl.fr/json/balises.json', timeout=(self.connect_timeout, self.read_timeout))
ffvl_stations = result.json()
for ffvl_station in ffvl_stations:
ffvl_id = None
try:
ffvl_id = ffvl_station['idBalise']
station = self.save_station(
ffvl_id,
ffvl_station['nom'],
ffvl_station['nom'],
ffvl_station['latitude'],
ffvl_station['longitude'],
Status.GREEN,
altitude=ffvl_station['altitude'],
url=ffvl_station['url'])
stations[station['_id']] = station
except ProviderException as e:
self.log.warn(f"Error while processing station '{ffvl_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{ffvl_id}': {e}")
except ProviderException as e:
self.log.warn(f'Error while processing stations: {e}')
except Exception as e:
self.log.exception(f'Error while processing stations: {e}')
try:
@retry(wait=wait_random(min=1, max=3), stop=(stop_after_delay(15)))
def request_data():
# data.ffvl.fr randomly returns an empty file instead the json doc
result = requests.get(
'http://data.ffvl.fr/json/relevesmeteo.json', timeout=(self.connect_timeout, self.read_timeout))
return result.json()
ffvl_measures = request_data()
ffvl_tz = tz.gettz('Europe/Paris')
for ffvl_measure in ffvl_measures:
station_id = None
try:
ffvl_id = ffvl_measure['idbalise']
station_id = self.get_station_id(ffvl_id)
if station_id not in stations:
raise ProviderException(f"Unknown station '{station_id}'")
station = stations[station_id]
measures_collection = self.measures_collection(station_id)
new_measures = []
key = arrow.get(ffvl_measure['date'], 'YYYY-MM-DD HH:mm:ss').replace(tzinfo=ffvl_tz).timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
ffvl_measure['directVentMoy'],
ffvl_measure['vitesseVentMoy'],
ffvl_measure['vitesseVentMax'],
temperature=ffvl_measure['temperature'],
humidity=ffvl_measure['hydrometrie'],
pressure=Pressure(qfe=ffvl_measure['pression'], qnh=None, qff=None)
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station, new_measures)
except ProviderException as e:
self.log.warn(f"Error while processing measures for station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing measures for station '{station_id}': {e}")
except ProviderException as e:
self.log.warn(f'Error while processing FFVL: {e}')
except Exception as e:
self.log.exception(f'Error while processing FFVL: {e}')
self.log.info('...Done!')
