def __init__(self,
worker=None,
api_key=None,
secret_key=None,
host=None,
logger=None,
verbose=None,
initialize_logging=True,
client=None,
config=None,
http_retries_config=None,
**kwargs):
# add backward compatibility support for old environment variables
backward_compatibility_support()
if config is not None:
self.config = config
else:
self.config = load()
if initialize_logging:
self.config.initialize_logging(
debug=kwargs.get('debug', False))
token_expiration_threshold_sec = self.config.get(
"auth.token_expiration_threshold_sec", 60)
super(Session, self).__init__(
token_expiration_threshold_sec=token_expiration_threshold_sec,
**kwargs)
self._verbose = verbose if verbose is not None else ENV_VERBOSE.get()
self._logger = logger
self.__access_key = api_key or ENV_ACCESS_KEY.get(
default=(self.config.get("api.credentials.access_key", None)
or self.default_key),
value_cb=lambda key, value: logger.info(
"Using environment access key {}={}".format(key, value)))
if not self.access_key:
raise ValueError(
"Missing access_key. Please set in configuration file or pass in session init."
)
self.__secret_key = secret_key or ENV_SECRET_KEY.get(
default=(self.config.get("api.credentials.secret_key", None)
or self.default_secret),
value_cb=lambda key, value: logger.info(
"Using environment secret key {}=********".format(key)))
if not self.secret_key:
raise ValueError(
"Missing secret_key. Please set in configuration file or pass in session init."
)
host = host or self.get_api_server_host(config=self.config)
if not host:
raise ValueError("host is required in init or config")
self.__host = host.strip("/")
http_retries_config = http_retries_config or self.config.get(
"api.http.retries", ConfigTree()).as_plain_ordered_dict()
http_retries_config["status_forcelist"] = self._retry_codes
self.__worker = worker or gethostname()
self.__max_req_size = self.config.get("api.http.max_req_size", None)
if not self.__max_req_size:
raise ValueError("missing max request size")
self.client = client or "api-{}".format(__version__)
# limit the reconnect retries, so we get an error if we are starting the session
http_no_retries_config = dict(**http_retries_config)
http_no_retries_config['connect'] = self._session_initial_connect_retry
self.__http_session = get_http_session_with_retry(
**http_no_retries_config)
# try to connect with the server
self.refresh_token()
# create the default session with many retries
self.__http_session = get_http_session_with_retry(
**http_retries_config)
# update api version from server response
try:
token_dict = TokenManager.get_decoded_token(self.token,
verify=False)
api_version = token_dict.get('api_version')
if not api_version:
api_version = '2.2' if token_dict.get(
'env', '') == 'prod' else Session.api_version
Session.api_version = str(api_version)
except (jwt.DecodeError, ValueError):
pass
# now setup the session reporting, so one consecutive retries will show warning
# we do that here, so if we have problems authenticating, we see them immediately
# notice: this is across the board warning omission
urllib_log_warning_setup(total_retries=http_retries_config.get(
'total', 0),
display_warning_after=3)
def __init__(
self,
worker=None,
api_key=None,
secret_key=None,
host=None,
logger=None,
verbose=None,
initialize_logging=True,
client=None,
config=None,
**kwargs
):
if config is not None:
self.config = config
else:
self.config = load()
if initialize_logging:
self.config.initialize_logging()
token_expiration_threshold_sec = self.config.get(
"auth.token_expiration_threshold_sec", 60
)
super(Session, self).__init__(
token_expiration_threshold_sec=token_expiration_threshold_sec, **kwargs
)
self._verbose = verbose if verbose is not None else ENV_VERBOSE.get()
self._logger = logger
self.__access_key = api_key or ENV_ACCESS_KEY.get(
default=self.config.get("api.credentials.access_key", None)
)
if not self.access_key:
raise ValueError(
"Missing access_key. Please set in configuration file or pass in session init."
)
self.__secret_key = secret_key or ENV_SECRET_KEY.get(
default=self.config.get("api.credentials.secret_key", None)
)
if not self.secret_key:
raise ValueError(
"Missing secret_key. Please set in configuration file or pass in session init."
)
host = host or ENV_HOST.get(default=self.config.get("api.host"))
if not host:
raise ValueError("host is required in init or config")
self.__host = host.strip("/")
http_retries_config = self.config.get(
"api.http.retries", ConfigTree()
).as_plain_ordered_dict()
http_retries_config["status_forcelist"] = self._retry_codes
self.__http_session = get_http_session_with_retry(**http_retries_config)
self.__worker = worker or gethostname()
self.__max_req_size = self.config.get("api.http.max_req_size")
if not self.__max_req_size:
raise ValueError("missing max request size")
self.client = client or "api-{}".format(__version__)
self.refresh_token()
# update api version from server response
try:
api_version = jwt.decode(self.token, verify=False).get('api_version', Session.api_version)
Session.api_version = str(api_version)
except (jwt.DecodeError, ValueError):
pass
def to_json(obj):
return HOCONConverter.to_json(ConfigTree(obj), compact=True, indent=1)
def __init__(self, config_tree=None):
if config_tree is None:
config_tree = ConfigTree()
self._config_tree = config_tree
self._namestack = []
def to_hocon(obj):
return HOCONConverter.to_hocon(ConfigTree(obj))
def __init__(self, *args, **kwargs):
# make sure we set the environment variable so the parent session opens the correct file
if kwargs.get('config_file'):
config_file = Path(os.path.expandvars(
kwargs.get('config_file'))).expanduser().absolute().as_posix()
kwargs['config_file'] = config_file
LOCAL_CONFIG_FILE_OVERRIDE_VAR.set(config_file)
if not Path(config_file).is_file():
raise ValueError(
"Could not open configuration file: {}".format(
config_file))
cpu_only = kwargs.get('cpu_only')
if cpu_only:
os.environ['CUDA_VISIBLE_DEVICES'] = os.environ[
'NVIDIA_VISIBLE_DEVICES'] = 'none'
if kwargs.get('gpus') and not os.environ.get('KUBERNETES_SERVICE_HOST') \
and not os.environ.get('KUBERNETES_PORT'):
# CUDA_VISIBLE_DEVICES does not support 'all'
if kwargs.get('gpus') == 'all':
os.environ.pop('CUDA_VISIBLE_DEVICES', None)
os.environ['NVIDIA_VISIBLE_DEVICES'] = kwargs.get('gpus')
else:
os.environ['CUDA_VISIBLE_DEVICES'] = os.environ[
'NVIDIA_VISIBLE_DEVICES'] = kwargs.get('gpus')
if kwargs.get('only_load_config'):
from clearml_agent.backend_api.config import load
self.config = load()
else:
super(Session, self).__init__(*args, **kwargs)
# set force debug mode, if it's on:
if Session.force_debug:
self.config["agent"]["debug"] = True
self.log = self.get_logger(__name__)
self.trace = kwargs.get('trace', False)
self._config_file = kwargs.get(
'config_file') or LOCAL_CONFIG_FILE_OVERRIDE_VAR.get()
if not self._config_file:
for f in reversed(LOCAL_CONFIG_FILES):
if os.path.exists(os.path.expanduser(os.path.expandvars(f))):
self._config_file = f
break
self.api_client = APIClient(session=self, api_version="2.5")
# HACK make sure we have python version to execute,
# if nothing was specific, use the one that runs us
def_python = ConfigValue(self.config, "agent.default_python")
if not def_python.get():
def_python.set("{version.major}.{version.minor}".format(
version=sys.version_info))
# HACK: backwards compatibility
if ENVIRONMENT_BACKWARD_COMPATIBLE.get():
os.environ['ALG_CONFIG_FILE'] = self._config_file
os.environ['SM_CONFIG_FILE'] = self._config_file
if not self.config.get('api.host', None) and self.config.get(
'api.api_server', None):
self.config['api']['host'] = self.config.get('api.api_server')
# initialize nvidia visibility variables
os.environ['CUDA_DEVICE_ORDER'] = "PCI_BUS_ID"
if os.environ.get('NVIDIA_VISIBLE_DEVICES'
) and not os.environ.get('CUDA_VISIBLE_DEVICES'):
# do not create CUDA_VISIBLE_DEVICES if it doesn't exist, it breaks TF/PyTotch CUDA detection
# os.environ['CUDA_VISIBLE_DEVICES'] = os.environ.get('NVIDIA_VISIBLE_DEVICES')
pass
elif os.environ.get('CUDA_VISIBLE_DEVICES'
) and not os.environ.get('NVIDIA_VISIBLE_DEVICES'):
os.environ['NVIDIA_VISIBLE_DEVICES'] = os.environ.get(
'CUDA_VISIBLE_DEVICES')
# override with environment variables
# cuda_version & cudnn_version are overridden with os.environ here, and normalized in the next section
for config_key, env_config in ENVIRONMENT_CONFIG.items():
# check if the propery is of a list:
if config_key.endswith('.0'):
if all(not i.get() for i in env_config.values()):
continue
parent = config_key.partition('.0')[0]
if not self.config[parent]:
self.config.put(parent, [])
self.config.put(
parent, self.config[parent] +
[ConfigTree((k, v.get()) for k, v in env_config.items())])
continue
value = env_config.get()
if not value:
continue
env_key = ConfigValue(self.config, config_key)
env_key.set(value)
# initialize cuda versions
try:
from clearml_agent.helper.package.requirements import RequirementsManager
agent = self.config['agent']
agent['cuda_version'], agent['cudnn_version'] = \
RequirementsManager.get_cuda_version(self.config) if not cpu_only else ('0', '0')
except Exception:
pass
# initialize worker name
worker_name = ConfigValue(self.config, "agent.worker_name")
if not worker_name.get():
worker_name.set(platform.node())
if not kwargs.get('only_load_config'):
self.create_cache_folders()
def tree(*args):
"""
Helper function for creating config trees
"""
return ConfigTree(args)
def __init__(self,
worker=None,
api_key=None,
secret_key=None,
host=None,
logger=None,
verbose=None,
initialize_logging=True,
client=None,
config=None,
**kwargs):
if config is not None:
self.config = config
else:
self.config = load()
if initialize_logging:
self.config.initialize_logging()
token_expiration_threshold_sec = self.config.get(
"auth.token_expiration_threshold_sec", 60)
super(Session, self).__init__(
token_expiration_threshold_sec=token_expiration_threshold_sec,
**kwargs)
self._verbose = verbose if verbose is not None else ENV_VERBOSE.get()
self._logger = logger
self.__access_key = api_key or ENV_ACCESS_KEY.get(
default=(self.config.get("api.credentials.access_key", None)
or "EGRTCO8JMSIGI6S39GTP43NFWXDQOW"))
if not self.access_key:
raise ValueError(
"Missing access_key. Please set in configuration file or pass in session init."
)
self.__secret_key = secret_key or ENV_SECRET_KEY.get(
default=(self.config.get("api.credentials.secret_key", None)
or "x!XTov_G-#vspE*Y(h$Anm&DIc5Ou-F)jsl$PdOyj5wG1&E!Z8"))
if not self.secret_key:
raise ValueError(
"Missing secret_key. Please set in configuration file or pass in session init."
)
host = host or ENV_HOST.get(default=self.config.get("api.host"))
if not host:
raise ValueError("host is required in init or config")
self.__host = host.strip("/")
http_retries_config = self.config.get(
"api.http.retries", ConfigTree()).as_plain_ordered_dict()
http_retries_config["status_forcelist"] = self._retry_codes
self.__http_session = get_http_session_with_retry(
**http_retries_config)
self.__worker = worker or gethostname()
self.__max_req_size = self.config.get("api.http.max_req_size")
if not self.__max_req_size:
raise ValueError("missing max request size")
self.client = client or "api-{}".format(__version__)
self.refresh_token()
import argparse
import os
import re
from typing import List
import requests
import pandas as pd
from pyhocon import ConfigFactory, ConfigTree
from tabulate import tabulate
from services_spec_generator.nginx_conf_parser.http_context import HttpContext
from services_spec_generator.nginx_config_merge import NginxMergedDumper
dir_path = os.path.dirname(os.path.realpath(__file__))
config = ConfigTree(ConfigFactory.parse_file(f'{dir_path}/config.conf'))
def get_hosts_ips(bind_zones_file_location):
map_ips = {} # {'192.168.0.1': ['example.com', 'work.example.com']}
map_hosts = {} # {'exmaple.com': '192.168.1.1'}
with open(bind_zones_file_location, 'r', encoding='utf-8') as file:
for line in file:
# for parse such record: d-1cdb1 IN A 10.69.0.241
zone_record = re.search(r'([^\s;]*)\s+IN\s+A\s+([\d\.]+)', line)
host = zone_record.group(1) if zone_record else None
ip = zone_record.group(2) if zone_record else None
if host is not None:
