def setup(self):
url = "{}:{}".format(self.host, self.port)
self.ccm = ConnectionManager()
self.connection = self.ccm.get_connection('crate')
# Added backoff_factor for retry interval between attempt of
# consecutive retries
backoff_factor = EnvReader(log=logging.getLogger(__name__).debug) \
.read(FloatVar('CRATE_BACKOFF_FACTOR', 0.0))
if self.connection is None:
try:
self.connection = client.connect([url],
error_trace=True,
backoff_factor=backoff_factor)
self.ccm.set_connection('crate', self.connection)
except Exception as e:
self.logger.warning(str(e), exc_info=True)
raise e
self.cursor = self.connection.cursor()
# TODO this reduce queries to crate,
# but only within a single API call to QUANTUMLEAP
# we need to think if we want to cache this information
# and save few msec for evey API call
self.db_version = self.get_db_version()
self.active_shards = EnvReader(log=logging.getLogger(__name__).debug) \
.read(StrVar('CRATE_WAIT_ACTIVE_SHARDS', '1'))
major = int(self.db_version.split('.')[0])
if major < 4:
logging.error("CRATE 4.x is the minimal version supported")
raise Exception("Unsupported CrateDB version")
def CrateTranslatorInstance():
r = EnvReader(log=logging.getLogger(__name__).info)
db_host = r.read(StrVar('CRATE_HOST', 'crate'))
db_port = r.read(IntVar('CRATE_PORT', 4200))
db_name = "ngsi-tsdb"
with CrateTranslator(db_host, db_port, db_name) as trans:
yield trans
def log():
r = EnvReader(log=logging.getLogger(__name__).info)
level = r.read(StrVar('LOGLEVEL', 'INFO')).upper()
logging.basicConfig(level=level,
format='%(asctime)s.%(msecs)03d '
'%(levelname)s:%(name)s:%(message)s '
'Thread ID: [%(thread)d] Process ID: [%(process)d]',
datefmt='%Y-%m-%d %I:%M:%S')
return logging.getLogger(__name__)
def default_backend() -> MaybeString:
cfg_reader = YamlReader(log=log().debug)
env_reader = EnvReader(log=log().debug)
config = cfg_reader.from_env_file(QL_CONFIG_ENV_VAR, defaults={})
config_backend = maybe_string_match(config, 'default-backend')
env_backend = env_reader.read(StrVar(QL_DEFAULT_DB_ENV_VAR, None))
return env_backend or config_backend or CRATE_BACKEND
def lookup_backend(fiware_service: str) -> MaybeString:
cfg_reader = YamlReader(log=log().debug)
env_reader = EnvReader(log=log().info)
config = cfg_reader.from_env_file(QL_CONFIG_ENV_VAR, defaults={})
tenant_backend = maybe_string_match(config, 'tenants', fiware_service,
'backend')
default_backend = maybe_string_match(config, 'default-backend')
env_backend = env_reader.read(StrVar(QL_DEFAULT_DB_ENV_VAR, CRATE_BACKEND))
return tenant_backend or env_backend or default_backend
def configured_insert_max_size_in_bytes() -> Optional[int]:
"""
Read the insert max size env var and return its value in bytes if
set to a parsable value or ``None`` otherwise. Notice if a value
is present but is garbage we still return ``None`` but we also
log a warning.
:return: the max size in bytes if available, ``None`` otherwise.
"""
env_reader = EnvReader(log=_log().debug)
parsed = env_reader.safe_read(BitSizeVar(INSERT_MAX_SIZE_VAR, None))
if parsed:
return int(parsed.to_Byte())
return None
class GeoCodingEnvReader:
"""
Helper class to encapsulate the reading of geo-coding env vars.
"""
def __init__(self):
self.env = EnvReader(log=logging.getLogger(__name__).info)
def use_geocoding(self) -> bool:
return self.env.read(BoolVar('USE_GEOCODING', False))
def redis_host(self) -> MaybeString:
return self.env.read(StrVar('REDIS_HOST', None))
def redis_port(self) -> int:
return self.env.read(IntVar('REDIS_PORT', 6379))
class SQLTranslatorConfig:
"""
Provide access to SQL Translator config values.
"""
def __init__(self, env: dict = os.environ):
self.store = EnvReader(var_store=env,
log=logging.getLogger(__name__).debug)
def default_limit(self) -> int:
var = IntVar(DEFAULT_LIMIT_VAR, default_value=FALLBACK_LIMIT)
return self.store.safe_read(var)
def keep_raw_entity(self) -> bool:
var = BoolVar(KEEP_RAW_ENTITY_VAR, False)
return self.store.safe_read(var)
def log_level() -> int:
"""
Read the log level to use from the ``LOGLEVEL`` environment variable.
If the variable isn't set, return the info level ID. If set but its
value isn't one of the strings recognised by the ``logging`` lib
(case-insensitive comparison), then return the info level ID again.
Otherwise return the corresponding log level ID.
:return: one of the log level IDs known to the ``logging`` lib.
"""
r = EnvReader()
level_name = r.safe_read(LOG_LEVEL_VAR).upper()
try:
return logging._nameToLevel[level_name]
except KeyError:
return logging.INFO
def successful_task_retention_period() -> int:
"""
How long to keep successfully executed tasks in the system. Past that
period, any successful task gets deleted.
:return: how long, in seconds, to keep successful tasks.
"""
return EnvReader().safe_read(SUCCESS_TTL_VAR)
def offload_to_work_queue() -> bool:
"""
Offload task execution to the work queue?
:return: `True` to offload tasks to the work queue; `False` to execute
them synchronously within the calling thread.
"""
return EnvReader().safe_read(OFFLOAD_WORK_VAR)
def failed_task_retention_period() -> int:
"""
How long to keep failed tasks in the system. Past that period, failed
tasks get deleted. Notice if you configure a task with retries, then
it gets flagged as "failed" only after all retries attempts have failed.
:return: how long, in seconds, to keep failed tasks.
"""
return EnvReader().safe_read(FAILURE_TTL_VAR)
def lookup_backend(fiware_service: str) -> MaybeString:
cfg_reader = YamlReader(log=log().debug)
env_reader = EnvReader(log=log().debug)
config = cfg_reader.from_env_file(QL_CONFIG_ENV_VAR, defaults={})
tenant_backend = maybe_string_match(config, 'tenants', fiware_service,
'backend')
return tenant_backend or default_backend()
class CacheEnvReader:
"""
Helper class to encapsulate the reading of geo-coding env vars.
"""
def __init__(self):
self.env = EnvReader(log=logging.getLogger(__name__).debug)
def redis_host(self) -> MaybeString:
return self.env.read(StrVar('REDIS_HOST', None))
def redis_port(self) -> int:
return self.env.read(IntVar('REDIS_PORT', 6379))
def default_ttl(self) -> int:
return self.env.read(IntVar('DEFAULT_CACHE_TTL', 60))
def cache_queries(self) -> bool:
return self.env.read(BoolVar('CACHE_QUERIES', False))
def read_env(self, env: dict = os.environ):
r = EnvReader(env, log=logging.getLogger(__name__).debug)
self.host = r.read(StrVar(CRATE_HOST_ENV_VAR, self.host))
self.port = r.read(IntVar(CRATE_PORT_ENV_VAR, self.port))
self.db_user = r.read(StrVar(CRATE_DB_USER_ENV_VAR, self.db_user))
self.db_pass = r.read(
StrVar(CRATE_DB_PASS_ENV_VAR, self.db_pass, mask_value=True))
# Added backoff_factor for retry interval between attempt of
# consecutive retries
self.backoff_factor = r.read(FloatVar('CRATE_BACKOFF_FACTOR', 0.0))
self.active_shards = r.read(StrVar('CRATE_WAIT_ACTIVE_SHARDS', '1'))
def db_con_factory(t: DbType) -> Any:
if t is DbType.CRATE:
r = EnvReader()
host = r.read(StrVar('CRATE_HOST', 'localhost'))
port = r.read(IntVar('CRATE_PORT', 4200))
return client.connect([f"{host}:{port}"], error_trace=True)
if t is DbType.TIMESCALE:
cfg = PostgresConnectionData()
cfg.read_env()
pg8000.paramstyle = "qmark"
cx = pg8000.connect(host=cfg.host,
port=cfg.port,
database=cfg.db_name,
user=cfg.db_user,
password=cfg.db_pass)
cx.autocommit = True
return cx
return None
def recover_from_enqueueing_failure() -> bool:
"""
Attempt to run tasks synchronously if the queue is temporarily not
available? When offloading task execution to the work queue, it could
happen that the queueing of a task fails, e.g. the queue backend is
down and the task can't be added to the work queue. In that case, if
this function returns ``True``, then QL tries to recover from the
error by executing the task synchronously in the calling thread.
On the other hand, if this function returns ``False``, then QL will
just raise an error.
Only take this setting into account if ``offload_to_work_queue`` is
``True``. (If ``False``, then tasks already get run synchronously.)
:return: ``True`` for try synchronous task execution on enqueueing
failure, ``False`` for raise an error instead.
"""
return EnvReader().safe_read(RECOVER_FROM_ENQUEUEING_FAILURE_VAR)
def use_flask() -> bool:
env_var = BoolVar('USE_FLASK', False)
return EnvReader().safe_read(env_var)
def max_retries() -> int:
"""
:return: how many times a failed task should be retried.
"""
return EnvReader().safe_read(MAX_RETRIES_VAR)
def log():
r = EnvReader(log=logging.getLogger(__name__).info)
level = r.read(StrVar('LOGLEVEL', 'INFO')).upper()
logging.basicConfig(level=level)
return logging.getLogger(__name__)
def __init__(self):
self.env = EnvReader(log=logging.getLogger(__name__).debug)
def __init__(self, env: dict = os.environ):
self.store = EnvReader(var_store=env,
log=logging.getLogger(__name__).info)
def setup():
r = EnvReader(log=logging.getLogger().debug)
level = r.read(StrVar('LOGLEVEL', 'INFO')).upper()
logger = logging.getLogger()
logger.setLevel(level)
logger.addHandler(default_handler)
