class Task:
def __init__(self, path):
self.path = path
def get_data_from_django(self):
url = f"http://127.0.0.1:8000/baiapp/drf{self.path}"
response = requests.get(url=url)
res = response.json()
for i in res[0:]:
if i['id'] == 1:
data = i
return data
def get_vnum_from_django(self):
data = self.get_data_from_django()
vnum = data['vnum']
return vnum
retry(
stop=stop_after_attempt(100),
wait=wait_exponential(multiplier=1, min=1, max=10),
before=before_log(logger, logging.INFO), #retry之前的日志等级
after=after_log(logger, logging.WARN), #retry之后的日志等级
)
def wait_vnum_up_django(self):
vnum = self.get_vnum_from_django()
# if vnum == 0:
# logger.error("阅读量为0")
# raise ValueError("阅读量为0")
@retry(
stop=stop_after_attempt(600), #尝试600次后就不尝试了
wait=wait_exponential(
multiplier=1, min=1,
max=10), #开始的时候等待 2^x * 1 秒，最少等待1秒，最多10秒，之后都是等待10秒
before=before_log(logger, logging.INFO),
after=after_log(logger, logging.WARN),
)
def get_vnum_is_growing(self):
vnum = self.get_vnum_from_django()
if (0 < vnum < 10):
logger.error("阅读量增长中")
raise ValueError("阅读量增长中")
elif vnum == 0:
logger.error("阅读量为0")
raise ValueError("阅读量为0")
return vnum
def is_finish_vnum(self):
self.wait_vnum_up_django()
vnum = self.get_vnum_is_growing()
if vnum == 10:
logger.info(f"阅读量完成{vnum}")
else:
logger.error(f"阅读量不到10，只有 {vnum} 遗憾")
return vnum
class PlatTask(HttpPlatBase):
"""description"""
def __init__(self,task_id,path):
self.task_id = task_id
self.times_out = 5
self.path = path
super().__init__(self.path)
'''向中台发起task结果轮询'''
def get_task_from_plat(self):
res = self.get_all()
return res
def get_task_state(self):
task = self.get_task_from_plat()
return task["state"]
@retry(
stop=stop_after_attempt(100),
wait=wait_exponential(multiplier=1, min=1, max=10),
before=before_log(logger, logging.INFO),
after=after_log(logger, logging.WARN),
)
def wait_for_task_start(self):
state = self.get_task_state()
if state == "not_exist":
logger.error("任务不存在")
raise ValueError("任务不存在")
@retry(
stop=stop_after_attempt(600),
wait=wait_exponential(multiplier=1, min=1, max=10),
before=before_log(logger, logging.INFO),
after=after_log(logger, logging.WARN),
)
def task_state(self):
state = self.get_task_state()
if state == "beginning":
logger.error("任务进行中")
raise ValueError("任务进行中")
return state
def is_task_successed(self):
self.wait_for_task_start()
state = self.task_state()
if state not in ("success", "failed"):
logger.error(f"task {self.task_id} 结果: {state}") #not_exist或beginning
else:
logger.info(f"task {self.task_id} 结果: {state}")
# result={"result":state}
return state #返回True或flase
def get_connection_from_config():
retries = CONF.database.max_retries
url = CONF.database.connection
try:
# TOTO(iafek): check why this call randomly fails
connection_scheme = urlparse.urlparse(url).scheme
LOG.debug('looking for %(name)r driver in %(namespace)r', {
'name': connection_scheme,
'namespace': _NAMESPACE
})
mgr = driver.DriverManager(_NAMESPACE, connection_scheme)
except Exception:
LOG.exception('Failed to get scheme %s.' % url)
return None
@tenacity.retry(wait=tenacity.wait_fixed(CONF.database.retry_interval),
stop=tenacity.stop_after_attempt(retries),
after=tenacity.after_log(LOG, log.WARN),
reraise=True)
def _get_connection():
"""Return an open connection to the database."""
conn = mgr.driver(url)
session = conn._engine_facade.get_session()
session.execute('SELECT 1;')
return conn
return _get_connection()
def decorator_f(self, *args, **kwargs):
retry_args = getattr(self, 'retry_args', None)
if retry_args is None:
return fun(self, *args, **kwargs)
multiplier = retry_args.get('multiplier', 1)
min_limit = retry_args.get('min', 1)
max_limit = retry_args.get('max', 1)
stop_after_delay = retry_args.get('stop_after_delay', 10)
tenacity_before_logger = tenacity.before_log(
self.log, logging.INFO) if self.log else None
tenacity_after_logger = tenacity.after_log(
self.log, logging.INFO) if self.log else None
default_kwargs = {
'wait':
tenacity.wait_exponential(multiplier=multiplier,
max=max_limit,
min=min_limit),
'retry':
tenacity.retry_if_exception(should_retry),
'stop':
tenacity.stop_after_delay(stop_after_delay),
'before':
tenacity_before_logger,
'after':
tenacity_after_logger,
}
return tenacity.retry(**default_kwargs)(fun)(self, *args,
**kwargs)
async def multi_tasks(cls,
tasks: Union[Iterable, Iterator],
to_do_func: Optional[Callable] = None,
concur_req: int = 4,
rate: float = 1.5,
logger: Optional[logging.Logger] = None):
'''
Template for multiTasking
TODO
1. asyncio.Semaphore
2. unit func
'''
cls.init_logger('UnsyncFetch', logger)
cls.retry_kwargs['after'] = after_log(cls.logger, logging.WARNING)
cls.http_download = retry(cls.http_download, **cls.retry_kwargs)
cls.ftp_download = retry(cls.ftp_download, **cls.retry_kwargs)
semaphore = asyncio.Semaphore(concur_req)
if to_do_func is None:
tasks = [
cls.fetch_file(semaphore, method, info, path, rate)
for method, info, path in tasks
]
else:
tasks = [
cls.fetch_file(semaphore, method, info, path,
rate).then(to_do_func)
for method, info, path in tasks
]
# return await asyncio.gather(*tasks)
return [
await fob
for fob in tqdm(asyncio.as_completed(tasks), total=len(tasks))
]
def pull(repo_dir: str, branch: str, revision: str) -> None:
"""Pull a revision from a branch of a remote repository into a local repository"""
@tenacity.retry(
stop=tenacity.stop_after_attempt(2),
reraise=True,
after=tenacity.after_log(logger, logging.DEBUG),
retry=tenacity.retry_if_exception_type(subprocess.TimeoutExpired),
)
def trigger_pull() -> None:
cmd = _build_hg_cmd(
"pull",
f"https://hg.mozilla.org/{branch}/".encode("ascii"),
r=revision.encode("ascii"),
debug=True,
)
p = subprocess.Popen(cmd, cwd=repo_dir)
try:
p.wait(timeout=180)
except subprocess.TimeoutExpired:
p.terminate()
p.wait()
raise
if p.returncode != 0:
raise Exception(
f"Error {p.returncode} when pulling {revision} from {branch}")
trigger_pull()
def batch_wait(self, tasks, timeout=300, wait_exp_multiplier=0.05, wait_exp_max=1.0):
"""
Wait until a list of task are completed. Expires after 'timeout' seconds.
Returns a tuple of list (pending_tasks, success_tasks, error_tasks).
Each list contains a couple (original_position, task) sorted by original_position asc
original_position gives the original index in the input tasks list parameter. This helps to keep the order.
"""
try:
positions = {}
pending_tasks = []
for pos, task in enumerate(tasks):
positions[task.pk] = pos
pending_tasks.append((pos, task))
success_tasks = []
error_tasks = []
retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max),
stop=stop_after_delay(timeout),
retry=retry_if_result(has_pending_tasks),
before=before_log(logger, logging.DEBUG),
after=after_log(logger, logging.DEBUG))
retryer(self._refresh_tasks_status, pending_tasks, success_tasks, error_tasks, positions)
except RetryError:
pass
return (sorted(pending_tasks, key=lambda v: v[0]),
sorted(success_tasks, key=lambda v: v[0]),
sorted(error_tasks, key=lambda v: v[0]))
def test_connection(
ctx,
max_tries=1, # seconds
wait_seconds=1,
):
"""Test database connection"""
@retry(
stop=stop_after_attempt(max_tries),
wait=wait_fixed(wait_seconds),
before=before_log(logger, logging.INFO),
after=after_log(logger, logging.WARN),
)
def init() -> None:
try:
db = SessionLocal()
# Try to create session to check if DB is awake
db.execute("SELECT 1")
ctx.database_connection = True
logger.info("Database connection is working")
except Exception as e:
ctx.database_connection = False
logger.error(e)
logger.error("Could not connect to database")
init()
class ShoutboxClient:
def __init__(self, settings: ShoutboxSettings):
self._settings = settings
self._prewritten = re.compile(
rf"({')+|('.join(settings.filter_phrases)})+", flags=re.I)
def _detect_prewritten(self, text: str) -> bool:
return bool(self._prewritten.search(text))
@cached_property
def enabled(self) -> bool:
return self._settings.url is not None
@tenacity.retry(
reraise=True,
retry=tenacity.retry_if_exception_type(requests.RequestException),
stop=tenacity.stop_after_attempt(3),
before_sleep=tenacity.before_sleep_log(logger, logger.level),
after=tenacity.after_log(logger, logger.level),
)
def make_request(self, data: ShoutboxRequest) -> ShoutboxResponse:
if self._settings.url is None:
raise RuntimeError("Shoutbox is disabled, so should not be here!")
response = requests.post(
self._settings.url.human_repr(),
json=data.dict(),
timeout=self._settings.read_timeout,
)
response.raise_for_status()
model = ShoutboxResponse.parse_obj(response.json())
if self._detect_prewritten(model.text):
raise ShoutboxPrewrittenDetectedError(
f"Detected shoutbox prewritten: '{model.text}'", )
return model
class GitUrlWatcher(SubTask):
def __init__(self, app_config: Dict):
super().__init__(name="git repo watcher")
self.watched_repos = []
watched_compose_files_config = app_config["main"]["watched_git_repositories"]
for config in watched_compose_files_config:
repo = GitRepo(
repo_id=config["id"],
repo_url=config["url"],
branch=config["branch"],
tags=config["tags"],
pull_only_files=config["pull_only_files"],
username=config["username"],
password=config["password"],
paths=config["paths"],
)
self.watched_repos.append(repo)
async def init(self) -> Dict:
description = await _init_repositories(self.watched_repos)
return description
@retry(
reraise=True,
stop=stop_after_attempt(NUMBER_OF_ATTEMPS),
wait=wait_random(min=1, max=MAX_TIME_TO_WAIT_S),
after=after_log(log, logging.DEBUG),
)
async def check_for_changes(self) -> Dict:
return await _check_repositories(self.watched_repos)
async def cleanup(self):
await _delete_repositories(self.watched_repos)
def retry(apply_func, retry_if, wait, stop, **kwargs):
retryer = Retrying(retry=retry_if,
wait=wait,
stop=stop,
before=before_log(logger, logging.DEBUG),
after=after_log(logger, logging.DEBUG),
**kwargs)
return retryer(apply_func)
def _retry_send(self, function: Callable, attempt=10, *args, **kwargs):
retry_configuration = tenacity.Retrying(
stop=tenacity.stop_after_attempt(attempt),
wait=tenacity.wait_fixed(3) + tenacity.wait_random(0, 2),
after=tenacity.after_log(logger, logger.level) if logger else None,
reraise=True,
)
return retry_configuration(function, *args, **kwargs)
def decorator(fun: T):
default_kwargs = {
'wait': tenacity.wait_exponential(multiplier=1, max=300),
'retry': retry_if_operation_in_progress(),
'before': tenacity.before_log(log, logging.DEBUG),
'after': tenacity.after_log(log, logging.DEBUG),
}
default_kwargs.update(**kwargs)
return cast(T, tenacity.retry(*args, **default_kwargs)(fun))
def _wrapper(*args, **kwargs):
r = tenacity.Retrying(
before_sleep=tenacity.before_sleep_log(LOG, logging.DEBUG),
after=tenacity.after_log(LOG, logging.DEBUG),
stop=tenacity.stop_after_attempt(retries),
reraise=True,
retry=tenacity.retry_if_exception_type(exceptions),
wait=wait)
return r.call(f, *args, **kwargs)
def decorator(fun: Callable):
default_kwargs = {
'wait': tenacity.wait_exponential(multiplier=1, max=100),
'retry': retry_if_temporary_quota(),
'before': tenacity.before_log(log, logging.DEBUG),
'after': tenacity.after_log(log, logging.DEBUG),
}
default_kwargs.update(**kwargs)
return tenacity.retry(*args, **default_kwargs)(fun)
def save_configuration(self, net_connect):
"""Save the device's configuration.
:param net_connect: a netmiko connection object.
:raises GenericSwitchNetmikoConfigError if saving the configuration
fails.
"""
# Junos configuration is transactional, and requires an explicit commit
# of changes in order for them to be applied. Since committing requires
# an exclusive lock on the configuration database, it can fail if
# another session has a lock. We use a retry mechanism to work around
# this.
class DBLocked(Exception):
"""Switch configuration DB is locked by another user."""
@tenacity.retry(
# Log a message after each failed attempt.
after=tenacity.after_log(LOG, logging.DEBUG),
# Reraise exceptions if our final attempt fails.
reraise=True,
# Retry on failure to commit the configuration due to the DB
# being locked by another session.
retry=(tenacity.retry_if_exception_type(DBLocked)),
# Stop after the configured timeout.
stop=tenacity.stop_after_delay(
int(self.ngs_config['ngs_commit_timeout'])),
# Wait for the configured interval between attempts.
wait=tenacity.wait_fixed(
int(self.ngs_config['ngs_commit_interval'])),
)
def commit():
try:
net_connect.commit()
except ValueError as e:
# Netmiko raises ValueError on commit failure, and appends the
# CLI output to the exception message. Raise a more specific
# exception for a locked DB, on which tenacity will retry.
DB_LOCKED_MSG = "error: configuration database locked"
if DB_LOCKED_MSG in str(e):
raise DBLocked(e)
raise
try:
commit()
except DBLocked as e:
msg = ("Reached timeout waiting for switch configuration DB lock. "
"Configuration might not be committed. Error: %s" % str(e))
LOG.error(msg)
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config), error=msg)
except ValueError as e:
msg = "Failed to commit configuration: %s" % e
LOG.error(msg)
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config), error=msg)
def __init__(self, SlackClientClass):
self.SlackClientClass = SlackClientClass
self.logger = get_logger('SlackClientWrapper')
self.standard_retrier = Retrying(
reraise=True,
wait=wait_fixed(2),
stop=stop_after_attempt(5),
after=after_log(logger=self.logger,
log_level=self.logger.getEffectiveLevel()),
retry=(retry_if_exception_type(ConnectionError)))
def _wrapper(*args, **kwargs):
r = tenacity.Retrying(sleep=tenacity.nap.sleep,
before_sleep=tenacity.before_sleep_log(
LOG, logging.DEBUG),
after=tenacity.after_log(LOG, logging.DEBUG),
stop=tenacity.stop_after_attempt(retries),
reraise=True,
retry=retry(retry_param),
wait=wait)
return r.call(f, *args, **kwargs)
def __init__(self, logger: Optional[logging.Logger] = None):
logger = logger or log
self.kwargs = dict(
retry=retry_if_exception_type(DatabaseError),
wait=wait_fixed(self.WAIT_SECS),
stop=stop_after_attempt(self.ATTEMPTS_COUNT),
after=after_log(logger, logging.WARNING),
retry_error_callback=raise_http_unavailable_error,
)
def __init__(self, strand_api_client):
self.strand_api_client = strand_api_client
self.logger = get_logger('StrandApiClientWrapper')
self.standard_retrier = Retrying(
reraise=True,
wait=wait_fixed(2),
stop=stop_after_attempt(5),
after=after_log(logger=self.logger,
log_level=self.logger.getEffectiveLevel()),
retry=retry_if_exception_type(StrandApiClientException))
def _wrapper(*args, **kwargs):
r = tenacity.Retrying(
before_sleep=tenacity.before_sleep_log(LOG, logging.DEBUG),
after=tenacity.after_log(LOG, logging.DEBUG),
stop=tenacity.stop_after_attempt(retries),
reraise=True,
retry=tenacity.retry_if_exception_type(exceptions),
wait=tenacity.wait_exponential(multiplier=interval,
min=0,
exp_base=backoff_rate))
return r.call(f, *args, **kwargs)
def wait_for_operation(operation_request,
test_success_fn,
timeout_sec=_WAIT_FOR_OPERATION_SEC,
wait_sec=_WAIT_FIXED_SEC):
retryer = tenacity.Retrying(
retry=(tenacity.retry_if_not_result(test_success_fn)
| tenacity.retry_if_exception_type()),
wait=tenacity.wait_fixed(wait_sec),
stop=tenacity.stop_after_delay(timeout_sec),
after=tenacity.after_log(logger, logging.DEBUG),
reraise=True)
return retryer(operation_request.execute)
class PostgresConnection:
def __init__(self, name: str, dsn: str, timeout=30000, autocommit=False):
self.name = name
self.listening = False
self.dsn = psycopg2.extensions.make_dsn(dsn, connect_timeout=timeout)
self.isolation = ISOLATION_AUTOCOMMIT if autocommit else ISOLATION_DEFAULT
self._connection = None
@tenacity.retry(
reraise=True,
wait=tenacity.wait_exponential(multiplier=RETRY_CONNECT_MULTIPLIER),
stop=tenacity.stop_after_attempt(RETRY_CONNECT_TRIES),
before=tenacity.before_log(Context.logger, logging.DEBUG),
after=tenacity.after_log(Context.logger, logging.DEBUG),
)
def _connect_db(self):
Context.logger.info(
f'Creating connection to PostgreSQL database "{self.name}"')
connection = psycopg2.connect(dsn=self.dsn)
connection.set_isolation_level(self.isolation)
# test connection
cursor = connection.cursor()
cursor.execute(query='SELECT * FROM document_queue;')
result = cursor.fetchall()
Context.logger.debug(f'Jobs in queue: {result}')
cursor.close()
connection.commit()
self._connection = connection
self.listening = False
def connect(self):
if not self._connection or self._connection.closed != 0:
self._connect_db()
@property
def connection(self):
self.connect()
return self._connection
def new_cursor(self, use_dict: bool = False):
return self.connection.cursor(
cursor_factory=psycopg2.extras.DictCursor if use_dict else None, )
def reset(self):
self.close()
self.connect()
def close(self):
if self._connection:
Context.logger.info(
f'Closing connection to PostgreSQL database "{self.name}"')
self._connection.close()
self._connection = None
def retry_api_call(func, config, logger=None, *args, **kwargs):
retry = tenacity.Retrying(
retry=retry_if_exception(lambda e: getattr(e, 'response', {}).get(
'Error', {}).get('Code', None) in config.exceptions
if e else False),
stop=stop_after_attempt(config.attempt),
wait=wait_exponential(multiplier=config.multiplier,
max=config.max_delay,
exp_base=config.exponential_base),
after=after_log(logger, logger.level) if logger else None,
reraise=True)
return retry(func, *args, **kwargs)
def retry(func):
tretry = tenacity.retry(
wait=(tenacity.wait_random_exponential(multiplier=1, min=0.1, max=10)),
after=tenacity.after_log(LOG, logging.WARNING),
)
def log_exc(*a, **k):
try:
return func(*a, **k)
except Exception:
LOG.exception("Exception calling %r" % (func, ))
raise
return tretry(log_exc)
def post(self, teams_webhook_url, message):
@retry(wait=wait_fixed(self.wait_time),
after=after_log(log, logging.WARN))
def post_with_retry(teams_webhook_url, message):
self._do_post(teams_webhook_url, message)
def simple_post(teams_webhook_url, message):
self._do_post(teams_webhook_url, message)
log.debug('The message that will be sent is: ' + message)
if self.retry:
post_with_retry(teams_webhook_url, message)
else:
simple_post(teams_webhook_url, message)
def test_02_custom_sec_format(self):
"""Test log formatting with custom int format.."""
log = unittest.mock.MagicMock(spec="logging.Logger.log")
logger = unittest.mock.MagicMock(spec="logging.Logger", log=log)
sec_format = "%.1f"
delay_since_first_attempt = 0.1
retry_state = test_tenacity.make_retry_state(self.previous_attempt_number, delay_since_first_attempt)
fun = after_log(logger=logger, log_level=self.log_level, sec_format=sec_format)
fun(retry_state)
log.assert_called_once_with(
self.log_level,
f"Finished call to '{_utils.get_callback_name(retry_state.fn)}' "
f"after {sec_format % retry_state.seconds_since_start}(s), "
f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it.",
)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.readthedocs.org/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
@tenacity.retry(
stop=tenacity.stop_after_attempt(100),
wait=tenacity.wait_random(min=2, max=5),
before=tenacity.before_log(logging.getLogger('tenacity.retry'),
logging.DEBUG),
before_sleep=tenacity.before_sleep_log(
logging.getLogger('tenacity.retry'), logging.INFO),
after=tenacity.after_log(logging.getLogger('tenacity.retry'),
logging.DEBUG))
def try_connect(db):
return db.connect()
with try_connect(engine) as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
with context.begin_transaction():
context.run_migrations()
connection.close()
async def run_with_retry(func: t.Callable[..., t.Awaitable[T]],
tries: t.Optional[int] = 2,
pause: t.Optional[int] = 15,
retry_exc: t.Union[
t.Type[Exception],
t.Sequence[t.Type[Exception]]] = Exception,
*args: t.Any,
**kwargs: t.Any) -> T:
return await tnc.AsyncRetrying( # type: ignore
wait=tnc.wait_fixed(pause),
stop=tnc.stop_after_attempt(tries),
retry=tnc.retry_if_exception_type(retry_exc),
reraise=True,
before=tnc.before_log(logger, logging.DEBUG),
after=tnc.after_log(logger, logging.DEBUG),
retry_error_cls=tnc.RetryError,
).call(func, *args, **kwargs)
def wait(self, timeout=60, wait_exp_multiplier=0.05, wait_exp_max=1.0):
"""
Wait until task is completed. Expires after 'timeout' seconds.
"""
try:
retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max),
stop=stop_after_delay(timeout),
retry=retry_if_result(is_pending_status),
before=before_log(logger, logging.DEBUG),
after=after_log(logger, logging.DEBUG))
retryer(self._refresh_status)
except RetryError:
raise TaskTimeout(self.data())
if is_error_status(self['status']):
raise TaskError(self.data())
return self
def upgrade_and_close():
""" Used in migration service program to discover, upgrade and close"""
for attempt in Retrying(wait=wait_fixed(5),
after=after_log(log, logging.ERROR)):
with attempt:
if not discover.callback():
raise Exception("Postgres db was not discover")
# FIXME: if database is not stampped!?
try:
info.callback()
upgrade.callback(revision="head")
info.callback()
except Exception:
log.exception("Unable to upgrade")
click.echo("I did my job here. Bye!")
async def managed_docker_compose(postgres_volume_name: str,
postgres_username: str,
postgres_password: str):
typer.echo("starting up database in localhost")
compose_file = Path.cwd() / "consistency" / "docker-compose.yml"
try:
subprocess.run(
f"docker-compose --file {compose_file} up --detach",
shell=True,
check=True,
cwd=compose_file.parent,
env={
**os.environ,
**{
"POSTGRES_DATA_VOLUME": postgres_volume_name
}
},
)
typer.echo(
f"database started: adminer available on http://127.0.0.1:18080/?pgsql=postgres&username={postgres_username}&db=simcoredb&ns=public"
)
@retry(
wait=wait_random(1, 3),
stop=stop_after_attempt(10),
after=after_log(log, logging.WARN),
)
async def postgres_responsive():
async with aiopg.create_pool(
f"dbname=simcoredb user={postgres_username} password={postgres_password} host=127.0.0.1"
) as pool:
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute("SELECT 1")
await postgres_responsive()
yield
finally:
subprocess.run(
f"docker-compose --file {compose_file} down",
shell=True,
check=True,
cwd=compose_file.parent,
)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.readthedocs.org/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = tenacity.Retrying(
stop=tenacity.stop_after_attempt(100),
wait=tenacity.wait_random(min=2, max=5),
before=tenacity.before_log(logging.getLogger("tenacity.retry"), logging.DEBUG),
before_sleep=tenacity.before_sleep_log(logging.getLogger("tenacity.retry"), logging.INFO),
after=tenacity.after_log(logging.getLogger("tenacity.retry"), logging.DEBUG)
).call(engine.connect)
context.configure(connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
