def __init__(self, logger: Optional[logging.Logger] = None):
logger = logger or log
self.kwargs = dict(
retry=retry_if_exception_type(DatabaseError),
wait=wait_fixed(self.WAIT_SECS),
stop=stop_after_attempt(self.ATTEMPTS_COUNT),
after=after_log(logger, logging.WARNING),
retry_error_callback=raise_http_unavailable_error,
)
def upgrade_and_close():
"""Used in migration service program to discover, upgrade and close"""
for attempt in Retrying(wait=wait_fixed(5), after=after_log(log, logging.ERROR)):
with attempt:
if not discover.callback():
raise Exception("Postgres db was not discover")
# FIXME: if database is not stampped!?
try:
info.callback()
upgrade.callback(revision="head")
info.callback()
except Exception: # pylint: disable=broad-except
log.exception("Unable to upgrade")
click.echo("I did my job here. Bye!")
async def managed_docker_compose(postgres_volume_name: str,
postgres_username: str,
postgres_password: str):
typer.echo("starting up database in localhost")
compose_file = Path.cwd() / "consistency" / "docker-compose.yml"
try:
subprocess.run(
["docker-compose", "--file", compose_file, "up", "--detach"],
shell=False,
check=True,
cwd=compose_file.parent,
env={
**os.environ,
**{
"POSTGRES_DATA_VOLUME": postgres_volume_name
}
},
)
typer.echo(
f"database started: adminer available on http://127.0.0.1:18080/?pgsql=postgres&username={postgres_username}&db=simcoredb&ns=public"
)
@retry(
wait=wait_random(1, 3),
stop=stop_after_attempt(10),
after=after_log(log, logging.WARN),
)
async def postgres_responsive():
async with aiopg.create_pool(
f"dbname=simcoredb user={postgres_username} password={postgres_password} host=127.0.0.1"
) as pool:
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute("SELECT 1")
await postgres_responsive()
yield
finally:
subprocess.run(
["docker-compose", "--file", compose_file, "down"],
shell=False,
check=True,
cwd=compose_file.parent,
)
def retry_api_call(func,
exceptions=('ThrottlingException',
'TooManyRequestsException'),
attempt=5,
multiplier=1,
max_delay=1800,
exp_base=2,
logger=None,
*args,
**kwargs):
retry = tenacity.Retrying(
retry=retry_if_exception(lambda e: getattr(e, 'response', {}).get(
'Error', {}).get('Code', None) in exceptions if e else False),
stop=stop_after_attempt(attempt),
wait=wait_exponential(multiplier=multiplier,
max=max_delay,
exp_base=exp_base),
after=after_log(logger, logger.level) if logger else None,
reraise=True)
return retry(func, *args, **kwargs)
from tenacity._asyncio import AsyncRetrying
from tenacity.after import after_log
from tenacity.retry import retry_if_exception_type
from tenacity.stop import stop_after_attempt, stop_after_delay
from tenacity.wait import wait_fixed
logger = logging.getLogger(__name__)
API_VERSION = "v0"
GARBAGE_COLLECTOR_INTERVAL = 1
SERVICE_DELETION_DELAY = 1
CHECK_BACKGROUND_RETRY_POLICY = dict(
stop=stop_after_attempt(2),
wait=wait_fixed(SERVICE_DELETION_DELAY + GARBAGE_COLLECTOR_INTERVAL),
retry=retry_if_exception_type(AssertionError),
after=after_log(logger, logging.INFO),
reraise=True,
)
@pytest.fixture
def mock_garbage_collector_task(mocker):
"""patch the setup of the garbage collector so we can call it manually"""
mocker.patch(
"simcore_service_webserver.resource_manager.module_setup.setup_garbage_collector",
return_value="",
)
@pytest.fixture
def mock_delete_data_folders_for_project(mocker):
"""Return the IP address for localhost"""
local_ip = default
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(("10.255.255.255", 1))
local_ip = s.getsockname()[0]
finally:
s.close()
return local_ip
@retry(
wait=wait_fixed(2),
stop=stop_after_attempt(10),
after=after_log(log, logging.WARNING),
)
def get_service_published_port(
service_name: str,
target_ports: Optional[Union[List[int], int]] = None) -> str:
# WARNING: ENSURE that service name exposes a port in
# Dockerfile file or docker-compose config file
# NOTE: retries since services can take some time to start
client = docker.from_env()
services = [
s for s in client.services.list() if str(s.name).endswith(service_name)
]
if not services:
raise RuntimeError(