class Config(BaseConfig):
loglevel = MapParameter(
default="info",
mappings={
# pylint: disable=used-before-assignment
"critical": logging.CRITICAL,
# pylint: disable=used-before-assignment
"error": logging.ERROR,
# pylint: disable=used-before-assignment
"warning": logging.WARNING,
# pylint: disable=used-before-assignment
"info": logging.INFO,
# pylint: disable=used-before-assignment
"debug": logging.DEBUG,
},
)
class activator(ConfigSection):
tos = IntParameter(min=0, max=255, default=0)
script_threads = IntParameter(default=10)
buffer_size = IntParameter(default=1048576)
connect_retries = IntParameter(default=3,
help="retries on immediate disconnect")
connect_timeout = IntParameter(
default=3, help="timeout after immediate disconnect")
http_connect_timeout = IntParameter(default=20)
http_request_timeout = IntParameter(default=30)
http_validate_cert = BooleanParameter(default=False)
class audit(ConfigSection):
command_ttl = SecondsParameter(default="1m")
login_ttl = SecondsParameter(default="1m")
reboot_ttl = SecondsParameter(default="0")
config_ttl = SecondsParameter(default="1y")
db_ttl = SecondsParameter(default="5y")
config_changed_ttl = SecondsParameter(default="1y")
class backup(ConfigSection):
keep_days = SecondsParameter(default="14d")
keep_weeks = SecondsParameter(default="12w")
keep_day_of_week = IntParameter(default="6")
keep_months = IntParameter(default="12")
keep_day_of_month = IntParameter(default="1")
class bi(ConfigSection):
language = StringParameter(default="en", help="Language BI interface")
query_threads = IntParameter(default=10)
extract_delay_alarms = SecondsParameter(default="1h")
clean_delay_alarms = SecondsParameter(default="1d")
reboot_interval = SecondsParameter(default="1M")
extract_delay_reboots = SecondsParameter(default="1h")
clean_delay_reboots = SecondsParameter(default="1d")
chunk_size = IntParameter(default=3000)
extract_window = SecondsParameter(default="1d")
enable_alarms = BooleanParameter(default=False)
enable_reboots = BooleanParameter(default=False)
enable_managedobjects = BooleanParameter(default=False)
enable_alarms_archive = BooleanParameter(default=False)
alarms_archive_policy = MapParameter(
default="weekly",
mappings={
"weekly": '{{doc["clear_timestamp"].strftime("y%Yw%W")}}',
"monthly": '{{doc["clear_timestamp"].strftime("y%Ym%m")}}',
"quarterly": '{{doc["clear_timestamp"].strftime("y%Y")}}'
'_quarter{{(doc["clear_timestamp"].month-1)//3 + 1}}',
"yearly": '{{doc["clear_timestamp"].strftime("y%Y")}}',
},
)
alarms_archive_batch_limit = IntParameter(default=10000)
brand = StringParameter(default="NOC")
class cache(ConfigSection):
vcinterfacescount = SecondsParameter(default="1h")
vcprefixes = SecondsParameter(default="1h")
cache_class = StringParameter(
default="noc.core.cache.mongo.MongoCache")
default_ttl = SecondsParameter(default="1d")
pool_size = IntParameter(default=8)
class card(ConfigSection):
language = StringParameter(default="en")
alarmheat_tooltip_limit = IntParameter(default=5)
class chwriter(ConfigSection):
batch_size = IntParameter(default=50000)
records_buffer = IntParameter(default=1000000)
batch_delay_ms = IntParameter(default=10000)
channel_expire_interval = SecondsParameter(default="5M")
suspend_timeout_ms = IntParameter(default=3000)
# Topic to listen
topic = StringParameter(default="chwriter")
# <address:port> of ClickHouse server to write
write_to = StringParameter()
max_in_flight = IntParameter(default=10)
class classifier(ConfigSection):
lookup_handler = HandlerParameter(
default="noc.services.classifier.rulelookup.RuleLookup")
default_interface_profile = StringParameter(default="default")
default_rule = StringParameter(default="Unknown | Default")
class clickhouse(ConfigSection):
rw_addresses = ServiceParameter(service="clickhouse", wait=True)
db = StringParameter(default="noc")
rw_user = StringParameter(default="default")
rw_password = SecretParameter()
ro_addresses = ServiceParameter(service="clickhouse", wait=True)
ro_user = StringParameter(default="readonly")
ro_password = SecretParameter()
request_timeout = SecondsParameter(default="1h")
connect_timeout = SecondsParameter(default="10s")
default_merge_tree_granularity = IntParameter(default=8192)
encoding = StringParameter(default="", choices=["", "deflate", "gzip"])
# Enable LowCardinality fileds
enable_low_cardinality = BooleanParameter(default=False)
# Cluster name for sharded/replicated configuration
# Matches appropriative <remote_servers> part
cluster = StringParameter()
# Cluster topology
# Expression in form
# <topology> ::= <shard> | <shard>,<topology>
# <shard> ::= [<weight>]<replicas>
# <weight> := <DIGITS>
# <replicas> := <DIGITS>
# Examples:
# 1 - non-replicated, non-sharded configuration
# 1,1 - 2 shards, non-replicated
# 2,2 - 2 shards, 2 replicas in each
# 3:2,2 - first shard has 2 replicas an weight 3,
# second shard has 2 replicas and weight 1
cluster_topology = StringParameter(default="1")
class cm(ConfigSection):
vcs_type = StringParameter(default="gridvcs",
choices=["hg", "CVS", "gridvcs"])
class collections(ConfigSection):
allow_sharing = BooleanParameter(default=True)
project_id = IntParameter(default=59)
class consul(ConfigSection):
token = SecretParameter()
connect_timeout = SecondsParameter(default="5s")
request_timeout = SecondsParameter(default="1h")
near_retry_timeout = IntParameter(default=1)
host = StringParameter(default="consul")
port = IntParameter(default=8500)
check_interval = SecondsParameter(default="10s")
check_timeout = SecondsParameter(default="1s")
release = SecondsParameter(default="1M")
session_ttl = SecondsParameter(default="10s")
lock_delay = SecondsParameter(default="20s")
retry_timeout = SecondsParameter(default="1s")
keepalive_attempts = IntParameter(default=5)
base = StringParameter(default="noc", help="kv lookup base")
class correlator(ConfigSection):
max_threads = IntParameter(default=20)
topology_rca_window = IntParameter(default=0)
oo_close_delay = SecondsParameter(default="20s")
discovery_delay = SecondsParameter(default="10M")
auto_escalation = BooleanParameter(default=True)
class customization(ConfigSection):
favicon_url = StringParameter(
default="/ui/web/img/logo_24x24_deep_azure.png")
logo_url = StringParameter(default="/ui/web/img/logo_white.svg")
logo_width = IntParameter(default=24)
logo_height = IntParameter(default=24)
branding_color = StringParameter(default="#ffffff")
branding_background_color = StringParameter(default="#34495e")
preview_theme = StringParameter(default="midnight")
class date_time_formats(StringParameter):
date_format = StringParameter(default="d.m.Y")
datetime_format = StringParameter(default="d.m.Y H:i:s")
month_day_format = StringParameter(default="F j")
time_format = StringParameter(default="H:i:s")
year_month_format = StringParameter(default="F Y")
class dcs(ConfigSection):
resolution_timeout = SecondsParameter(default="5M")
class discovery(ConfigSection):
max_threads = IntParameter(default=20)
sample = IntParameter(default=0)
class dns(ConfigSection):
warn_before_expired = SecondsParameter(default="30d")
class escalator(ConfigSection):
max_threads = IntParameter(default=5)
retry_timeout = SecondsParameter(default="60s")
tt_escalation_limit = IntParameter(default=10)
ets = SecondsParameter(default="60s")
wait_tt_check_interval = SecondsParameter(default="60s")
sample = IntParameter(default=0)
class features(ConfigSection):
use_uvlib = BooleanParameter(default=False)
cp = BooleanParameter(default=True)
sentry = BooleanParameter(default=False)
traefik = BooleanParameter(default=False)
cpclient = BooleanParameter(default=False)
telemetry = BooleanParameter(
default=False,
help="Enable internal telemetry export to Clickhouse")
consul_healthchecks = BooleanParameter(
default=True,
help="While registering serive in consul also register health check"
)
service_registration = BooleanParameter(
default=True, help="Permit consul self registration")
pypy = BooleanParameter(default=False)
forensic = BooleanParameter(default=False)
class fm(ConfigSection):
active_window = SecondsParameter(default="1d")
keep_events_wo_alarm = IntParameter(default=0)
keep_events_with_alarm = IntParameter(default=-1)
alarm_close_retries = IntParameter(default=5)
outage_refresh = SecondsParameter(default="60s")
total_outage_refresh = SecondsParameter(default="60s")
enable_rca_neighbor_cache = BooleanParameter(default=False)
class geocoding(ConfigSection):
order = StringParameter(default="yandex,google")
yandex_key = SecretParameter(default="")
yandex_apikey = SecretParameter(default="")
google_key = SecretParameter(default="")
google_language = StringParameter(default="en")
negative_ttl = SecondsParameter(default="7d",
help="Period then saving bad result")
class gis(ConfigSection):
ellipsoid = StringParameter(default="PZ-90")
enable_osm = BooleanParameter(default=True)
enable_google_sat = BooleanParameter(default=False)
enable_google_roadmap = BooleanParameter(default=False)
tile_size = IntParameter(default=256, help="Tile size 256x256")
tilecache_padding = IntParameter(default=0)
global_n_instances = IntParameter(default=1)
class grafanads(ConfigSection):
db_threads = IntParameter(default=10)
class http_client(ConfigSection):
connect_timeout = SecondsParameter(default="10s")
request_timeout = SecondsParameter(default="1h")
user_agent = StringParameter(default="noc")
buffer_size = IntParameter(default=128 * 1024)
max_redirects = IntParameter(default=5)
ns_cache_size = IntParameter(default=1000)
resolver_ttl = SecondsParameter(default="3s")
http_port = IntParameter(default=80)
https_port = IntParameter(default=443)
validate_certs = BooleanParameter(default=False,
help="Have to be set as True")
class initial(ConfigSection):
admin_user_name = StringParameter(default="admin")
admin_password = StringParameter(default="admin")
admin_email = StringParameter(default="*****@*****.**")
installation_name = StringParameter(default="Unconfigured installation")
instance = IntParameter(default=0)
language = StringParameter(default="en")
language_code = StringParameter(default="en")
class layout(ConfigSection):
ring_ring_edge = IntParameter(default=150)
ring_chain_edge = IntParameter(default=150)
ring_chain_spacing = IntParameter(default=100)
tree_horizontal_step = IntParameter(default=100)
tree_vertical_step = IntParameter(default=100)
tree_max_levels = IntParameter(default=4)
spring_propulsion_force = FloatParameter(default=1.5)
spring_edge_force = FloatParameter(default=1.2)
spring_bubble_force = FloatParameter(default=2.0)
spring_edge_spacing = IntParameter(default=190)
spring_iterations = IntParameter(default=50)
listen = StringParameter(default="auto:0")
log_format = StringParameter(default="%(asctime)s [%(name)s] %(message)s")
thread_stack_size = IntParameter(default=0)
gitlab_url = StringParameter("https://code.getnoc.com/")
class logging(ConfigSection):
log_api_calls = BooleanParameter(default=False)
log_sql_statements = BooleanParameter(default=False)
class login(ConfigSection):
methods = StringParameter(default="local")
session_ttl = SecondsParameter(default="7d")
language = StringParameter(default="en")
restrict_to_group = StringParameter(default="")
single_session_group = StringParameter(default="")
mutual_exclusive_group = StringParameter(default="")
idle_timeout = SecondsParameter(default="1w")
pam_service = StringParameter(default="noc")
radius_secret = SecretParameter(default="noc")
radius_server = StringParameter()
user_cookie_ttl = IntParameter(default=1)
register_last_login = BooleanParameter(default=True)
class mailsender(ConfigSection):
smtp_server = StringParameter()
smtp_port = IntParameter(default=25)
use_tls = BooleanParameter(default=False)
helo_hostname = StringParameter(default="noc")
from_address = StringParameter(default="*****@*****.**")
smtp_user = StringParameter()
smtp_password = SecretParameter()
class memcached(ConfigSection):
addresses = ServiceParameter(service="memcached",
wait=True,
full_result=True)
pool_size = IntParameter(default=8)
default_ttl = SecondsParameter(default="1d")
class mongo(ConfigSection):
addresses = ServiceParameter(service="mongo", wait=True)
db = StringParameter(default="noc")
user = StringParameter()
password = SecretParameter()
rs = StringParameter()
retries = IntParameter(default=20)
timeout = SecondsParameter(default="3s")
retry_writes = BooleanParameter(default=False)
app_name = StringParameter()
max_idle_time = SecondsParameter(default="60s")
class mrt(ConfigSection):
max_concurrency = IntParameter(default=50)
enable_command_logging = BooleanParameter(default=False)
node = socket.gethostname()
class nbi(ConfigSection):
max_threads = IntParameter(default=10)
objectmetrics_max_interval = SecondsParameter(default="3h")
class nsqd(ConfigSection):
addresses = ServiceParameter(service="nsqd",
wait=True,
near=True,
full_result=False)
http_addresses = ServiceParameter(service="nsqdhttp",
wait=True,
near=True,
full_result=False)
pub_retries = IntParameter(default=5)
pub_retry_delay = FloatParameter(default=1)
mpub_messages = IntParameter(default=10000)
mpub_size = IntParameter(default=1048576)
topic_mpub_rate = IntParameter(default=10)
ch_chunk_size = IntParameter(default=4000)
connect_timeout = SecondsParameter(default="3s")
request_timeout = SecondsParameter(default="30s")
reconnect_interval = IntParameter(default=15)
compression = StringParameter(choices=["", "deflate", "snappy"],
default="")
compression_level = IntParameter(default=6)
max_in_flight = IntParameter(default=1)
class nsqlookupd(ConfigSection):
addresses = ServiceParameter(service="nsqlookupd",
wait=True,
near=True,
full_result=False)
http_addresses = ServiceParameter(service="nsqlookupdhttp",
wait=True,
full_result=False)
class path(ConfigSection):
smilint = StringParameter()
smidump = StringParameter()
dig = StringParameter()
vcs_path = StringParameter(default="/usr/local/bin/hg")
repo = StringParameter(default="/var/repo")
backup_dir = StringParameter(default="/var/backup")
etl_import = StringParameter(default="/var/lib/noc/import")
ssh_key_prefix = StringParameter(default="etc/noc_ssh")
cp_new = StringParameter(default="/var/lib/noc/cp/crashinfo/new")
bi_data_prefix = StringParameter(default="/var/lib/noc/bi")
babel_cfg = StringParameter(default="etc/babel.cfg")
babel = StringParameter(default="./bin/pybabel")
pojson = StringParameter(default="./bin/pojson")
collection_fm_mibs = StringParameter(default="collections/fm.mibs/")
supervisor_cfg = StringParameter(default="etc/noc_services.conf")
legacy_config = StringParameter(default="etc/noc.yml")
cythonize = StringParameter(default="./bin/cythonize")
npkg_root = StringParameter(default="/var/lib/noc/var/pkg")
card_template_path = StringParameter(
default="services/card/templates/card.html.j2")
pm_templates = StringParameter(default="templates/ddash/")
custom_path = StringParameter()
mib_path = StringParameter(default="/var/mib")
class pg(ConfigSection):
addresses = ServiceParameter(service="postgres",
wait=True,
near=True,
full_result=False)
db = StringParameter(default="noc")
user = StringParameter()
password = SecretParameter()
connect_timeout = IntParameter(default=5)
class ping(ConfigSection):
throttle_threshold = FloatParameter()
restore_threshold = FloatParameter()
tos = IntParameter(min=0, max=255, default=0)
# Recommended send buffer size, 4M by default
send_buffer = IntParameter(default=4 * 1048576)
# Recommended receive buffer size, 4M by default
receive_buffer = IntParameter(default=4 * 1048576)
# DataStream request limit
ds_limit = IntParameter(default=1000)
class pmwriter(ConfigSection):
batch_size = IntParameter(default=2500)
metrics_buffer = IntParameter(default=50000)
read_from = StringParameter(default="pmwriter")
write_to = StringParameter(default="influxdb")
write_to_port = IntParameter(default=8086)
max_delay = FloatParameter(default="1.0")
class proxy(ConfigSection):
http_proxy = StringParameter(default=os.environ.get("http_proxy"))
https_proxy = StringParameter(default=os.environ.get("https_proxy"))
ftp_proxy = StringParameter(default=os.environ.get("ftp_proxy"))
pool = StringParameter(default=os.environ.get("NOC_POOL", ""))
class redis(ConfigSection):
addresses = ServiceParameter(service="redis",
wait=True,
full_result=True)
db = IntParameter(default=0)
default_ttl = SecondsParameter(default="1d")
class rpc(ConfigSection):
retry_timeout = StringParameter(default="0.1,0.5,1,3,10,30")
sync_connect_timeout = SecondsParameter(default="20s")
sync_request_timeout = SecondsParameter(default="1h")
sync_retry_timeout = FloatParameter(default=1.0)
sync_retry_delta = FloatParameter(default=2.0)
sync_retries = IntParameter(default=5)
async_connect_timeout = SecondsParameter(default="20s")
async_request_timeout = SecondsParameter(default="1h")
class sae(ConfigSection):
db_threads = IntParameter(default=20)
activator_resolution_retries = IntParameter(default=5)
activator_resolution_timeout = SecondsParameter(default="2s")
class scheduler(ConfigSection):
max_threads = IntParameter(default=20)
submit_threshold_factor = IntParameter(default=10)
max_chunk_factor = IntParameter(default=1)
updates_per_check = IntParameter(default=4)
cache_default_ttl = SecondsParameter(default="1d")
autointervaljob_interval = SecondsParameter(default="1d")
autointervaljob_initial_submit_interval = SecondsParameter(
default="1d")
class script(ConfigSection):
timeout = SecondsParameter(default="2M",
help="default sa script script timeout")
session_idle_timeout = SecondsParameter(default="1M",
help="default session timeout")
caller_timeout = SecondsParameter(default="1M")
calling_service = StringParameter(default="script")
secret_key = StringParameter(default="12345")
class selfmon(ConfigSection):
enable_managedobject = BooleanParameter(default=True)
managedobject_ttl = IntParameter(default=30)
enable_task = BooleanParameter(default=False)
task_ttl = IntParameter(default=30)
enable_inventory = BooleanParameter(default=False)
inventory_ttl = IntParameter(default=30)
enable_fm = BooleanParameter(default=False)
fm_ttl = IntParameter(default=30)
class sentry(ConfigSection):
url = StringParameter(default="")
class syslogcollector(ConfigSection):
listen = StringParameter(default="0.0.0.0:514")
enable_reuseport = BooleanParameter(default=True)
enable_freebind = BooleanParameter(default=False)
# DataStream request limit
ds_limit = IntParameter(default=1000)
class tgsender(ConfigSection):
token = SecretParameter()
retry_timeout = IntParameter(default=2)
use_proxy = BooleanParameter(default=False)
class threadpool(ConfigSection):
idle_timeout = SecondsParameter(default="30s")
shutdown_timeout = SecondsParameter(default="1M")
timezone = StringParameter(default="Europe/Moscow")
class traceback(ConfigSection):
reverse = BooleanParameter(default=True)
class trapcollector(ConfigSection):
listen = StringParameter(default="0.0.0.0:162")
enable_reuseport = BooleanParameter(default=True)
enable_freebind = BooleanParameter(default=False)
# DataStream request limit
ds_limit = IntParameter(default=1000)
class web(ConfigSection):
theme = StringParameter(default="gray")
api_row_limit = IntParameter(default=0)
api_arch_alarm_limit = IntParameter(default=4 * 86400)
max_upload_size = IntParameter(default=16777216)
language = StringParameter(default="en")
install_collection = BooleanParameter(default=False)
max_threads = IntParameter(default=10)
macdb_window = IntParameter(default=4 * 86400)
enable_remote_system_last_extract_info = BooleanParameter(
default=False)
class datasource(ConfigSection):
chunk_size = IntParameter(default=1000)
max_threads = IntParameter(default=10)
default_ttl = SecondsParameter(default="1h")
class datastream(ConfigSection):
enable_administrativedomain = BooleanParameter(default=False)
enable_administrativedomain_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for Adm. Domain datastream (Mongo greater 3.6 needed)",
)
enable_alarm = BooleanParameter(default=False)
enable_alarm_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for Alarm datastream (Mongo greater 3.6 needed)"
)
enable_cfgping = BooleanParameter(default=True)
enable_cfgping_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for CfgPing datastream (Mongo greater 3.6 needed)",
)
enable_cfgsyslog = BooleanParameter(default=True)
enable_cfgsyslog_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for CfgSyslog datastream (Mongo greater 3.6 needed)",
)
enable_cfgtrap = BooleanParameter(default=True)
enable_cfgtrap_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for CfgTrap datastream (Mongo greater 3.6 needed)",
)
enable_dnszone = BooleanParameter(default=False)
enable_dnszone_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for DNS Zone datastream (Mongo greater 3.6 needed)",
)
enable_managedobject = BooleanParameter(default=False)
enable_managedobject_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for ManagedObject datastream (Mongo greater 3.6 needed)",
)
enable_resourcegroup = BooleanParameter(default=False)
enable_resourcegroup_wait = BooleanParameter(
default=True,
help=
"Activate Wait Mode for ResourceGroup datastream (Mongo greater 3.6 needed)",
)
class help(ConfigSection):
base_url = StringParameter(default="https://docs.getnoc.com")
branch = StringParameter(default="microservices")
language = StringParameter(default="en")
class tests(ConfigSection):
# List of pyfilesystem URLs holding intial data
fixtures_paths = ListParameter(item=StringParameter(),
default=["tests/data"])
# List of pyfilesystem URLs holding event classification samples
events_paths = ListParameter(item=StringParameter())
# List of pyfilesystem URLs holding beef test cases
beef_paths = ListParameter(item=StringParameter())
class peer(ConfigSection):
enable_ripe = BooleanParameter(default=True)
enable_arin = BooleanParameter(default=True)
enable_radb = BooleanParameter(default=True)
prefix_list_optimization = BooleanParameter(default=True)
prefix_list_optimization_threshold = IntParameter(default=1000)
max_prefix_length = IntParameter(default=24)
rpsl_inverse_pref_style = BooleanParameter(default=False)
class metrics(ConfigSection):
default_hist = ListParameter(
item=FloatParameter(),
default=[0.001, 0.005, 0.01, 0.05, 0.5, 1.0, 5.0, 10.0])
enable_mongo_hist = BooleanParameter(default=False)
mongo_hist = ListParameter(
item=FloatParameter(),
default=[0.001, 0.005, 0.01, 0.05, 0.5, 1.0, 5.0, 10.0])
enable_postgres_hist = BooleanParameter(default=False)
postgres_hist = ListParameter(
item=FloatParameter(),
default=[0.001, 0.005, 0.01, 0.05, 0.5, 1.0, 5.0, 10.0])
default_quantiles = ListParameter(item=FloatParameter(),
default=[0.5, 0.9, 0.95])
default_quantiles_epsilon = 0.01
default_quantiles_window = 60
default_quantiles_buffer = 100
enable_mongo_quantiles = BooleanParameter(default=False)
enable_postgres_quantiles = BooleanParameter(default=False)
# pylint: disable=super-init-not-called
def __init__(self):
self.setup_logging()
@property
def pg_connection_args(self):
"""
PostgreSQL database connection arguments
suitable to pass to psycopg2.connect
"""
return {
"host": self.pg.addresses[0].host,
"port": self.pg.addresses[0].port,
"database": self.pg.db,
"user": self.pg.user,
"password": self.pg.password,
}
@property
def mongo_connection_args(self):
"""
Mongo connection arguments. Suitable to pass to
pymongo.connect and mongoengine.connect
"""
if not hasattr(self, "_mongo_connection_args"):
self._mongo_connection_args = {
"db": self.mongo.db,
"username": self.mongo.user,
"password": self.mongo.password,
}
if self.mongo.app_name:
self._mongo_connection_args["appname"] = self.mongo.app_name
if self.mongo.retry_writes:
self._mongo_connection_args["retryWrites"] = True
has_credentials = self.mongo.user or self.mongo.password
if has_credentials:
self._mongo_connection_args[
"authentication_source"] = self.mongo.db
hosts = self.mongo.addresses
if self.mongo.rs:
self._mongo_connection_args["replicaSet"] = self.mongo.rs
self._mongo_connection_args[
"readPreference"] = "secondaryPreferred"
elif len(hosts) > 1:
raise ValueError("Replica set name must be set")
if self.mongo.max_idle_time:
self._mongo_connection_args[
"maxIdleTimeMS"] = self.mongo.max_idle_time * 1000
url = ["mongodb://"]
if has_credentials:
url += [
"%s:%s@" % (urllib_quote(
self.mongo.user), urllib_quote(self.mongo.password))
]
url += [",".join(str(h) for h in hosts)]
url += ["/%s" % self.mongo.db]
self._mongo_connection_args["host"] = "".join(url)
if self.metrics.enable_mongo_hist:
from noc.core.mongo.monitor import MongoCommandSpan
self._mongo_connection_args["event_listeners"] = [
MongoCommandSpan()
]
return self._mongo_connection_args
def setup_logging(self, loglevel=None):
"""
Create new or setup existing logger
"""
if not loglevel:
loglevel = self.loglevel
logger = logging.getLogger()
if len(logger.handlers):
# Logger is already initialized
fmt = logging.Formatter(self.log_format, None)
for h in logging.root.handlers:
if isinstance(h, logging.StreamHandler):
h.stream = sys.stdout
h.setFormatter(fmt)
logging.root.setLevel(loglevel)
else:
# Initialize logger
logging.basicConfig(stream=sys.stdout,
format=self.log_format,
level=loglevel)
logging.captureWarnings(True)
@property
def ch_cluster_topology(self):
if not hasattr(self, "_ch_cluster_topology"):
shards = []
for s in self.clickhouse.cluster_topology.split(","):
s = s.strip()
if ":" in s:
weight, replicas = s.split(":")
else:
weight, replicas = 1, s
shards += [CHClusterShard(int(replicas), int(weight))]
self._ch_cluster_topology = shards
return self._ch_cluster_topology
def get_ch_topology_type(self):
"""
Detect ClickHouse topology type
:return: Any of
* CH_UNCLUSTERED
* CH_REPLICATED
* CH_SHARDED
"""
topo = self.ch_cluster_topology
if len(topo) == 1:
if topo[0].replicas == 1:
return CH_UNCLUSTERED
else:
return CH_REPLICATED
else:
return CH_SHARDED
def get_customized_paths(self, *args, **kwargs):
"""
Check for customized path for given repo path.
Repo path may be given in os.path.join-style components.
Returns list of possible paths. One of elements is always repo path,
while other may be custom counterpart, if exists.
:param prefer_custom: True - customized path first, False - repo path first
:param args: Path or path components in os.path.join-style
:return: List of possible paths
"""
prefer_custom = kwargs.get("prefer_custom", False)
rpath = os.path.join(*args)
if not self.path.custom_path:
return [rpath]
cpath = os.path.join(self.path.custom_path, *args)
if os.path.exists(cpath):
if prefer_custom:
return [cpath, rpath]
else:
return [rpath, cpath]
return [rpath]
def get_hist_config(self, name):
"""
Get configuration for hist `name`. Returns list of times or None, if hist is disabled
:param name: Hist name
:return: List of hist config or None
"""
# Check hist is enabled
if not getattr(self.metrics, "enable_%s_hist" % name, False):
return None
# Get config
cfg = getattr(self.metrics, "%s_hist" % name)
if cfg:
return cfg
# Fallback to defaults
return self.metrics.default_hist or None
def get_quantiles_config(self, name):
"""
Check if quantile is enabled
:return: True if quantile is enabled
"""
# Check quantiles is enabled
return getattr(self.metrics, "enable_%s_quantiles" % name, False)
class memcached(ConfigSection):
addresses = ServiceParameter(service="memcached",
wait=True,
full_result=True)
pool_size = IntParameter(default=8)
default_ttl = SecondsParameter(default="1d")
class discovery(ConfigSection):
max_threads = IntParameter(default=20)
sample = IntParameter(default=0)
class grafanads(ConfigSection):
db_threads = IntParameter(default=10)
class collections(ConfigSection):
allow_sharing = BooleanParameter(default=True)
project_id = IntParameter(default=59)
class correlator(ConfigSection):
max_threads = IntParameter(default=20)
topology_rca_window = IntParameter(default=0)
oo_close_delay = SecondsParameter(default="20s")
discovery_delay = SecondsParameter(default="10M")
auto_escalation = BooleanParameter(default=True)
class backup(ConfigSection):
keep_days = SecondsParameter(default="14d")
keep_weeks = SecondsParameter(default="12w")
keep_day_of_week = IntParameter(default="6")
keep_months = IntParameter(default="12")
keep_day_of_month = IntParameter(default="1")
class pg(ConfigSection):
addresses = ServiceParameter(service="postgres", wait=True, near=True, full_result=False)
db = StringParameter(default="noc")
user = StringParameter()
password = SecretParameter()
connect_timeout = IntParameter(default=5)
class trapcollector(ConfigSection):
listen = StringParameter(default="0.0.0.0:162")
enable_reuseport = BooleanParameter(default=True)
enable_freebind = BooleanParameter(default=False)
# DataStream request limit
ds_limit = IntParameter(default=1000)
class datasource(ConfigSection):
chunk_size = IntParameter(default=1000)
max_threads = IntParameter(default=10)
default_ttl = SecondsParameter(default="1h")
class tgsender(ConfigSection):
token = SecretParameter()
retry_timeout = IntParameter(default=2)
use_proxy = BooleanParameter(default=False)
class sae(ConfigSection):
db_threads = IntParameter(default=20)
activator_resolution_retries = IntParameter(default=5)
activator_resolution_timeout = SecondsParameter(default="2s")
class redis(ConfigSection):
addresses = ServiceParameter(service="redis", wait=True, full_result=True)
db = IntParameter(default=0)
default_ttl = SecondsParameter(default="1d")
class mrt(ConfigSection):
max_concurrency = IntParameter(default=50)
enable_command_logging = BooleanParameter(default=False)
class card(ConfigSection):
language = StringParameter(default="en")
alarmheat_tooltip_limit = IntParameter(default=5)
class nbi(ConfigSection):
max_threads = IntParameter(default=10)
objectmetrics_max_interval = SecondsParameter(default="3h")
class cache(ConfigSection):
vcinterfacescount = SecondsParameter(default="1h")
vcprefixes = SecondsParameter(default="1h")
cache_class = StringParameter(default="noc.core.cache.mongo.MongoCache")
default_ttl = SecondsParameter(default="1d")
pool_size = IntParameter(default=8)