def __init__(self, *args, **kwargs):
super(ConsulBackend, self).__init__(*args, **kwargs)
if self.consul is None:
raise ImproperlyConfigured(CONSUL_MISSING)
self._init_from_params(**parse_url(self.url))
def generate_docker_compose(context):
import toml
import jinja2
from sqlalchemy.engine import url
from kombu.utils.url import parse_url
config = "production"
configuration = toml.load(CONFIG_PATH.format(config=config))
db_url = configuration["SQLALCHEMY_DATABASE_URI"]
u = url.make_url(db_url)
mongo_info = configuration["MONGODB_SETTINGS"]
broker_url = configuration["CELERY_BROKER_URL"]
broker_info = parse_url(broker_url)
env = jinja2.Environment(
loader=jinja2.FileSystemLoader("tasks/app/templates/configurations"))
for temp_path, path in zip(
["docker-compose.yml.template", "init.js.template"],
["docker-compose.yml", "scripts/mongo/init.js"],
):
template = env.get_template(temp_path)
template.stream(
mq_user=broker_info["userid"]
if broker_info["userid"] else "guest",
mq_passwd=broker_info["password"]
if broker_info["password"] else "guest",
mongo_db=mongo_info["db"],
mongo_user=mongo_info["username"],
mongo_passwd=mongo_info["password"],
db_username=u.username,
db_password=u.password,
db_name=u.database,
).dump(path)
def __init__(self, *args, **kwargs):
super(ConsulBackend, self).__init__(*args, **kwargs)
if self.consul is None:
raise ImproperlyConfigured(CONSUL_MISSING)
self._init_from_params(**parse_url(self.url))
def test_default_broker_url():
url = parse_url(settings.BROKER_URL)
assert url['transport'] == 'amqp'
assert url['hostname'] == 'rabbitmq'
assert url['userid'] == 'guest'
assert url['password'] == 'guest'
assert url['virtual_host'] == '/'
def test_parse_url():
assert parse_url('amqp://*****:*****@localhost:5672/my/vhost') == {
'transport': 'amqp',
'userid': 'user',
'password': '******',
'hostname': 'localhost',
'port': 5672,
'virtual_host': 'my/vhost',
}
def test_broker_url_with_special_characters():
settings.BROKER_URL = 'amqp://*****:*****@ns:ibl3#@rabbitmq:5672//'
url = parse_url(settings.BROKER_URL)
assert url['transport'] == 'amqp'
assert url['hostname'] == 'rabbitmq'
assert url['port'] == 5672
assert url['userid'] == 'guest'
assert url['password'] == 'a@ns:ibl3#'
assert url['virtual_host'] == '/'
def test_parse_url(self):
result = parse_url('amqp://*****:*****@localhost:5672/my/vhost')
self.assertDictEqual(result, {
'transport': 'amqp',
'userid': 'user',
'password': '******',
'hostname': 'localhost',
'port': 5672,
'virtual_host': 'my/vhost',
})
def __init__(self, url=None, table_name=None, *args, **kwargs):
super(DynamoDBBackend, self).__init__(*args, **kwargs)
self.url = url
self.table_name = table_name or self.table_name
if not boto3:
raise ImproperlyConfigured(
'You need to install the boto3 library to use the '
'DynamoDB backend.')
aws_credentials_given = False
aws_access_key_id = None
aws_secret_access_key = None
if url is not None:
scheme, region, port, username, password, table, query = \
parse_url(url)
aws_access_key_id = username
aws_secret_access_key = password
access_key_given = aws_access_key_id is not None
secret_key_given = aws_secret_access_key is not None
if access_key_given != secret_key_given:
raise ImproperlyConfigured(
'You need to specify both the Access Key ID '
'and Secret.')
aws_credentials_given = access_key_given
if region == 'localhost':
# We are using the downloadable, local version of DynamoDB
self.endpoint_url = 'http://localhost:{}'.format(port)
self.aws_region = 'us-east-1'
logger.warning(
'Using local-only DynamoDB endpoint URL: {}'.format(
self.endpoint_url))
else:
self.aws_region = region
self.read_capacity_units = int(
query.get('read', self.read_capacity_units))
self.write_capacity_units = int(
query.get('write', self.write_capacity_units))
self.table_name = table or self.table_name
self._available_fields = (self._key_field, self._value_field,
self._timestamp_field)
self._client = None
if aws_credentials_given:
self._get_client(access_key_id=aws_access_key_id,
secret_access_key=aws_secret_access_key)
def test_parse_url(self):
result = parse_url('amqp://*****:*****@localhost:5672/my/vhost')
self.assertDictEqual(
result, {
'transport': 'amqp',
'userid': 'user',
'password': '******',
'hostname': 'localhost',
'port': 5672,
'virtual_host': 'my/vhost',
})
def test_ssl_parameters():
url = 'rediss://*****:*****@host:6379/0?'
querystring = urlencode({
'ssl_cert_reqs': 'CERT_REQUIRED',
'ssl_ca_certs': '/var/ssl/myca.pem',
'ssl_certfile': '/var/ssl/server-cert.pem',
'ssl_keyfile': '/var/ssl/priv/worker-key.pem',
})
kwargs = parse_url(url + querystring)
assert kwargs['transport'] == 'rediss'
assert kwargs['ssl']['ssl_cert_reqs'] == ssl.CERT_REQUIRED
assert kwargs['ssl']['ssl_ca_certs'] == '/var/ssl/myca.pem'
assert kwargs['ssl']['ssl_certfile'] == '/var/ssl/server-cert.pem'
assert kwargs['ssl']['ssl_keyfile'] == '/var/ssl/priv/worker-key.pem'
kombu.utils.url.ssl_available = False
kwargs = parse_url(url + querystring)
assert kwargs['ssl']['ssl_cert_reqs'] is None
kombu.utils.url.ssl_available = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.consul is None:
raise ImproperlyConfigured(CONSUL_MISSING)
#
# By default, for correctness, we use a client connection per
# operation. If set, self.one_client will be used for all operations.
# This provides for the original behaviour to be selected, and is
# also convenient for mocking in the unit tests.
#
self.one_client = None
self._init_from_params(**parse_url(self.url))
def _parse_config(config: Dict):
db_url = url.make_url(config["SQLALCHEMY_DATABASE_URI"])
broker_url = parse_url(config["CELERY_BROKER_URL"])
config["DB_URL_INFO"] = {
"username": db_url.username,
"password": db_url.password,
"database": db_url.database,
"host": db_url.host,
"port": db_url.port,
}
config["BROKER_URL_INFO"] = {
"username": broker_url["userid"],
"password": broker_url["password"],
}
def __init__(self, url=None, table_name=None, *args, **kwargs):
super(DynamoDBBackend, self).__init__(*args, **kwargs)
self.url = url
self.table_name = table_name or self.table_name
if not boto3:
raise ImproperlyConfigured(
'You need to install the boto3 library to use the '
'DynamoDB backend.')
aws_credentials_given = False
aws_access_key_id = None
aws_secret_access_key = None
if url is not None:
scheme, region, port, username, password, table, query = \
parse_url(url)
aws_access_key_id = username
aws_secret_access_key = password
access_key_given = aws_access_key_id is not None
secret_key_given = aws_secret_access_key is not None
if access_key_given != secret_key_given:
raise ImproperlyConfigured(
'You need to specify both the Access Key ID '
'and Secret.')
aws_credentials_given = access_key_given
if region == 'localhost':
# We are using the downloadable, local version of DynamoDB
self.endpoint_url = 'http://localhost:{}'.format(port)
self.aws_region = 'us-east-1'
logger.warning(
'Using local-only DynamoDB endpoint URL: {}'.format(
self.endpoint_url
)
)
else:
self.aws_region = region
self.read_capacity_units = int(
query.get(
'read',
self.read_capacity_units
)
)
self.write_capacity_units = int(
query.get(
'write',
self.write_capacity_units
)
)
self.table_name = table or self.table_name
self._available_fields = (
self._key_field,
self._value_field,
self._timestamp_field
)
self._client = None
if aws_credentials_given:
self._get_client(
access_key_id=aws_access_key_id,
secret_access_key=aws_secret_access_key
)
def __init__(self, url=None, table_name=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.url = url
self.table_name = table_name or self.table_name
if not boto3:
raise ImproperlyConfigured(
'You need to install the boto3 library to use the '
'DynamoDB backend.')
aws_credentials_given = False
aws_access_key_id = None
aws_secret_access_key = None
if url is not None:
scheme, region, port, username, password, table, query = \
parse_url(url)
aws_access_key_id = username
aws_secret_access_key = password
access_key_given = aws_access_key_id is not None
secret_key_given = aws_secret_access_key is not None
if access_key_given != secret_key_given:
raise ImproperlyConfigured(
'You need to specify both the Access Key ID '
'and Secret.')
aws_credentials_given = access_key_given
if region == 'localhost':
# We are using the downloadable, local version of DynamoDB
self.endpoint_url = f'http://localhost:{port}'
self.aws_region = 'us-east-1'
logger.warning(
'Using local-only DynamoDB endpoint URL: {}'.format(
self.endpoint_url))
else:
self.aws_region = region
# If endpoint_url is explicitly set use it instead
_get = self.app.conf.get
config_endpoint_url = _get('dynamodb_endpoint_url')
if config_endpoint_url:
self.endpoint_url = config_endpoint_url
self.read_capacity_units = int(
query.get('read', self.read_capacity_units))
self.write_capacity_units = int(
query.get('write', self.write_capacity_units))
ttl = query.get('ttl_seconds', self.time_to_live_seconds)
if ttl:
try:
self.time_to_live_seconds = int(ttl)
except ValueError as e:
logger.error(f'TTL must be a number; got "{ttl}"',
exc_info=e)
raise e
self.table_name = table or self.table_name
self._available_fields = (self._key_field, self._value_field,
self._timestamp_field)
self._client = None
if aws_credentials_given:
self._get_client(access_key_id=aws_access_key_id,
secret_access_key=aws_secret_access_key)
def parse_transport(app):
return parse_url(app.conf.broker_url)["transport"]
