def _get_connetcion(self, correlation_id):
connection = self.__connection_resolver.resolve(correlation_id)
# Check for connection
if connection is None:
raise ConfigException(correlation_id, "NO_CONNECTION",
"Connection for REST client is not defined")
else:
# Check for type
protocol = connection.get_protocol('http')
if 'http' != protocol and 'https' != protocol:
raise ConfigException(
correlation_id, "WRONG_PROTOCOL",
"Protocol is not supported by REST connection"
).with_details("protocol", protocol)
# Check for host
elif connection.get_host() is None:
raise ConfigException(
correlation_id, "NO_HOST",
"No host is configured in REST connection")
# Check for port
elif connection.get_port() == 0:
raise ConfigException(
correlation_id, "NO_PORT",
"No port is configured in REST connection")
return connection
def validate_connections(self, correlation_id, connections):
if connections == None or len(connections) == 0:
return ConfigException(correlation_id, "NO_CONNECTION",
"Database connection is not set")
for connection in connections:
error = self.validate_connection(correlation_id, connection)
def open(self, correlation_id: Optional[str]):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
connection = self.__connection_resolver.resolve(correlation_id)
if connection is None:
raise ConfigException(correlation_id, 'NO_CONNECTION',
'Connection is not configured')
credential = self.__credential_resolver.lookup(correlation_id)
options = {
# connect_timeout: self.__timeout,
# max_attempts: self.__retries,
'retry_on_timeout': True,
}
if connection.get_uri():
options['url'] = connection.get_uri()
else:
options['host'] = connection.get_host() or 'localhost'
options['port'] = connection.get_port() or 6379
if credential is not None:
options['password'] = credential.get_password()
self.__client = redis.Redis(**options)
def open(self, correlation_id: Optional[str]):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
connections = self.__connection_resolver.resolve_all(correlation_id)
if len(connections) == 0:
raise ConfigException(correlation_id, 'NO_CONNECTION',
'Connection is not configured')
servers: List[str] = []
for connection in connections:
host = connection.get_host()
port = connection.get_port() or 11211
servers.append(f'{host}:{port}')
options = {
# TODO: this options have not support by driver, but can execute by cmd driver method
# 'maxKeySize': self.__max_key_size,
# 'maxExpiration': self.__max_expiration,
# 'maxValue': self.__max_value, # driver don't have this config
# 'retries': self.__retries,
# 'remove': self.__remove,
'retry_attempts': self.
__failures, # driver automatically remove dead servers from the pool (by attemps)
'max_pool_size': self.__pool_size,
'connect_timeout': self.__reconnect / 1000,
'timeout': self.__timeout / 1000,
'retry_timeout': self.__retry / 1000,
'pool_idle_timeout': self.__idle / 1000,
'default_noreply': False
}
self.__client = pymemcache.HashClient(servers=servers, **options)
def load(self, correlation_id: Optional[str]) -> List[T]:
"""
Loads data items from external JSON file.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: loaded items
"""
# If doesn't exist then consider empty data
if self.__path is None:
raise ConfigException(correlation_id, "NO_PATH",
"Data file path is not set")
if not os.path.isfile(self.__path):
return []
try:
with open(self.__path, 'r') as file:
data = file.read()
list_data = JsonConverter.to_nullable_map(data)
arr = ArrayConverter.list_to_array(list_data)
return arr
except Exception as ex:
raise FileException(correlation_id, "READ_FAILED", "Failed to read data file: " + str(ex)) \
.with_cause(ex)
def read_from_file(correlation_id, path, parameters):
"""
Reads container configuration from JSON or YAML file.
The type of the file is determined by file extension.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param path: a path to component configuration file.
:param parameters: values to parameters the configuration or null to skip parameterization.
:return: the read container configuration
"""
if path == None:
raise ConfigException(correlation_id, "NO_PATH",
"Missing config file path")
index = path.rfind('.')
ext = path[index + 1:].lower() if index > 0 else ''
if ext == "json":
return ContainerConfigReader.read_from_json_file(
correlation_id, path, parameters)
elif ext == "yaml":
return ContainerConfigReader.read_from_yaml_file(
correlation_id, path, parameters)
# By default read as JSON
return ContainerConfigReader.read_from_json_file(
correlation_id, path, parameters)
def resolve(self, correlation_id: Optional[str]) -> ConfigParams:
"""
Resolves connection options from connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: resolved options or error
"""
connections = self._connection_resolver.resolve_all(correlation_id)
if len(connections) > 0 and not self._cluster_supported:
raise ConfigException(
correlation_id, "MULTIPLE_CONNECTIONS_NOT_SUPPORTED",
"Multiple (cluster) connections are not supported")
# Validate connections
for connection in connections:
self._validate_connection(correlation_id, connection)
credential = self._credential_resolver.lookup(correlation_id)
# Validate credential
self._validate_credential(correlation_id, credential)
options = self._compose_options(connections, credential, self._options)
return options
def open(self, correlation_id: Optional[str]):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.is_open():
return
connection = self.__connection_resolver.resolve(correlation_id)
if connection is None:
raise ConfigException(correlation_id, 'NO_CONNECTION',
'Connection is not configured')
uri = connection.get_as_string('uri')
options = {
'request_timeout': self.__timeout,
'dead_timeout': self.__reconnect,
'max_retries': self.__max_retries
}
self.__client = Elasticsearch(hosts=[uri], kwargs=options)
try:
self.__create_index_if_needed(correlation_id, True)
self.__timer = SetInterval(self.dump, self._interval)
self.__timer.start()
except Exception as err:
raise err
def __validate_connections(self, correlation_id: Optional[str],
connections: List[ConnectionParams]):
if connections is None or len(connections) == 0:
return ConfigException(correlation_id, "NO_CONNECTION",
"Database connection is not set")
for connection in connections:
error = self.__validate_connection(correlation_id, connection)
def __validate_connections(self, correlation_id: Optional[str],
connections: List[ConnectionParams]):
if not connections or len(connections) == 0:
raise ConfigException(correlation_id, "NO_CONNECTION",
"Database connection is not set")
for connection in connections:
self.__validate_connection(correlation_id, connection)
def __validate_connection(self,
correlation_id: Optional[str],
connection: ConnectionParams,
credential: CredentialParams = None):
# Sometimes when we use https we are on an internal network and do not want to have to deal with security.
# When we need a https connection and we don't want to pass credentials, flag is 'credential.internal_network',
# this flag just has to be present and non null for this functionality to work.
if connection is None:
raise ConfigException(correlation_id, "NO_CONNECTION",
"HTTP connection is not set")
uri = connection.get_as_string('uri')
if uri is not None:
return None
protocol = connection.get_protocol_with_default("http")
if protocol != "http" and 'https' != protocol:
raise ConfigException(correlation_id,
"WRONG_PROTOCOL",
"Protocol is not supported by REST connection") \
.with_details("protocol", protocol)
host = connection.get_as_string('host')
if host is None:
raise ConfigException(correlation_id, "NO_HOST",
"Connection host is not set")
port = connection.get_as_integer('port')
if port == 0:
raise ConfigException(correlation_id, "NO_PORT",
"Connection port is not set")
# Check HTTPS credentials
if protocol == 'https':
# Check for credential
if credential is None or credential.length() == 0:
raise ConfigException(
correlation_id, 'NO_CREDENTIAL',
'SSL certificates are not configured for HTTPS protocol')
else:
# Sometimes when we use https we are on an internal network and do not want to have to deal with
# security. When we need a https connection and we don't want to pass credentials,
# flag is 'credential.internal_network', this flag just has to be present and non null for this
# functionality to work.
if credential.get_as_nullable_string(
'internal_network') is None:
if credential.get_as_nullable_string(
'ssl_key_file') is None:
raise ConfigException(
correlation_id, 'NO_SSL_KEY_FILE',
'SSL key file is not configured in credentials')
elif credential.get_as_nullable_string(
'ssl_crt_file') is None:
raise ConfigException(
correlation_id, 'NO_SSL_CRT_FILE',
'SSL crt file is not configured in credentials')
return None
def validate_connection(self, correlation_id, connection):
uri = connection.get_uri()
if uri == None:
return None
host = connection.get_host()
if host == None:
return ConfigException(correlation_id, "NO_HOST",
"Connection host is not set")
port = connection.get_port()
if port == 0:
return ConfigException(correlation_id, "NO_PORT",
"Connection port is not set")
database = connection.get_as_nullable_string("database")
if database == None:
return ConfigException(correlation_id, "NO_DATABASE",
"Connection database is not set")
def __validate_connection(self, correlation_id: Optional[str],
connection: ConnectionParams):
uri = connection.get_uri()
if uri:
return
host = connection.get_host()
if not host:
raise ConfigException(correlation_id, "NO_HOST",
"Connection host is not set")
port = connection.get_port()
if port == 0:
raise ConfigException(correlation_id, "NO_PORT",
"Connection port is not set")
database = connection.get_as_nullable_string('database')
if not database:
raise ConfigException(correlation_id, "NO_DATABASE",
"Connection database is not set")
def _validate_connection(self, correlation_id: Optional[str],
connection: ConnectionParams):
"""
Validates connection parameters.
This method can be overriden in child classes.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param connection: connection parameters to be validated
:return: error or `None` if validation was successful
"""
if connection is None:
raise ConfigException(
correlation_id, "NO_CONNECTION",
"Connection parameters are not set is not set")
# URI usually contains all information
uri = connection.get_uri()
if uri is not None: return None
protocol = connection.get_protocol_with_default(self._default_protocol)
if protocol is None:
raise ConfigException(correlation_id, "NO_PROTOCOL",
"Connection protocol is not set")
if self._supported_protocols is not None and protocol in self._supported_protocols:
raise ConfigException(
correlation_id, "UNSUPPORTED_PROTOCOL",
"The protocol " + protocol + " is not supported")
host = connection.get_host()
if host is None:
raise ConfigException(correlation_id, "NO_HOST",
"Connection host is not set")
port = connection.get_port_with_default(self._default_port)
if port == 0:
raise ConfigException(correlation_id, "NO_PORT",
"Connection port is not set")
return None
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
try:
if config is not None or config.contains_key("path"):
self.path = config.get_as_string("path")
except AttributeError:
raise ConfigException(None, "NO_PATH", "Data file path is not set")
def _get_credential(self, correlation_id, connection):
# Credentials are not required unless HTTPS is used
if connection.get_protocol('http') != 'https':
return
credential = self.__credential_resolver.lookup(correlation_id)
# Check for connection
if credential is None:
raise ConfigException(
correlation_id, "NO_CREDENTIAL",
"SSL certificates are not configured for HTTPS protocol")
else:
if credential.get_as_nullable_string('ssl_key_file') is None:
raise ConfigException(
correlation_id, "NO_SSL_KEY_FILE",
"SSL key file is not configured in credentials")
elif credential.get_as_nullable_string('ssl_crt_file') is None:
raise ConfigException(
correlation_id, "NO_SSL_CRT_FILE",
"SSL crt file is not configured in credentials")
return credential
def from_config(config: ConfigParams) -> 'ComponentConfig':
"""
Creates a new instance of ComponentConfig based on section from container configuration.
:param config: component parameters from container configuration
:return: a newly created ComponentConfig
"""
descriptor = Descriptor.from_string(config.get_as_nullable_string("descriptor"))
type = TypeDescriptor.from_string(config.get_as_nullable_string("type"))
if descriptor is None and type is None:
raise ConfigException(None, "BAD_CONFIG", "Component configuration must have descriptor or type")
return ComponentConfig(descriptor, type, config)
def open(self, correlation_id: Optional[str]):
"""
Opens a connection using the parameters resolved by the referenced connection resolver and creates a REST server (service) using the set options and parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.is_open():
return
connection = self.__connection_resolver.resolve(correlation_id)
if connection is None:
raise ConfigException(correlation_id, "NO_CONNECTION",
"Connection for REST client is not defined")
self.__uri = connection.get_as_string('uri')
# verify https with bottle
certfile = None
keyfile = None
if connection.get_as_string_with_default('protocol',
'http') == 'https':
certfile = connection.get_as_nullable_string('ssl_crt_file')
keyfile = connection.get_as_nullable_string('ssl_key_file')
# Create instance of bottle application
self.__service = SessionMiddleware(
bottle.Bottle(catchall=True, autojson=True)).app
self.__service.config['catchall'] = True
self.__service.config['autojson'] = True
# Enable CORS requests
self.__service.add_hook('after_request', self.__enable_cors)
self.__service.add_hook('after_request', self.__do_maintance)
self.__service.add_hook('after_request', self.__no_cache)
self.__service.add_hook('before_request', self.__add_compatibility)
# Register routes
# self.__perform_registrations()
def start_server():
self.__service.run(server=self.__server, debug=self._debug)
# self.__perform_registrations()
host = connection.get_as_string('host')
port = connection.get_as_integer('port')
# Starting service
try:
self.__server = SSLCherryPyServer(host=host,
port=port,
certfile=certfile,
keyfile=keyfile)
# Start server in thread
Thread(target=start_server).start()
# Time for start server
time.sleep(0.01)
# Give 2 sec for initialization
self.__connection_resolver.register(correlation_id)
self.__logger.debug(
correlation_id,
f"Opened REST service at {self.__uri}",
)
self.__perform_registrations()
except Exception as ex:
self.__server = None
raise ConnectionException(correlation_id, 'CANNOT_CONNECT', 'Opening REST service failed') \
.wrap(ex).with_details('url', self.__uri)