def __init__(self, name, config, config_no_sensitive):
self.logger = getLogger(self.__class__.__name__)
self.name = name
self.config = config
self.engine_name = config['engine'] # self.engine.name is 'mysql' while 'self.engine_name' is mysql+pymysql
# Safe for printing out to logs, any sensitive data has been shadowed
self.config_no_sensitive = config_no_sensitive
_extra = {}
# Postgres-only
if self.engine_name.startswith('mysql'):
_extra['pool_recycle'] = 600
elif self.engine_name.startswith('postgres'):
_extra['connect_args'] = {'application_name': get_component_name()}
extra = self.config.get('extra') # Optional, hence .get
_extra.update(parse_extra_into_dict(extra))
engine_url = engine_def.format(**config)
self.engine = create_engine(engine_url, pool_size=int(config['pool_size']), **_extra)
event.listen(self.engine, 'checkin', self.on_checkin)
event.listen(self.engine, 'checkout', self.on_checkout)
event.listen(self.engine, 'connect', self.on_connect)
event.listen(self.engine, 'first_connect', self.on_first_connect)
def _get_engine(self, args):
engine_url = odb.engine_def.format(engine=args.odb_type, username=args.odb_user,
password=args.odb_password, host=args.odb_host, port=args.odb_port,
db_name=args.odb_db_name)
connect_args = {'application_name':util.get_component_name('enmasse')} if args.odb_type == 'postgresql' else {}
return sqlalchemy.create_engine(engine_url, connect_args=connect_args)
def test_create_headers_json(self):
cid = rand_string()
now = datetime.utcnow().isoformat()
requests_module = _FakeRequestsModule()
config = self._get_config()
config['data_format'] = DATA_FORMAT.JSON
config['transport'] = URL_TYPE.PLAIN_HTTP
wrapper = HTTPSOAPWrapper(config, requests_module)
user_headers = {
rand_string(): rand_string(),
rand_string(): rand_string()
}
headers = wrapper._create_headers(cid, user_headers, now)
eq_(headers.pop('X-Zato-CID'), cid)
eq_(headers.pop('X-Zato-Msg-TS'), now)
eq_(headers.pop('X-Zato-Component'), get_component_name())
eq_(headers.pop('Content-Type'), CONTENT_TYPE.JSON)
# Anything that is left must be user headers
self.assertDictEqual(headers, user_headers)
def test_create_headers_soap12(self):
cid = rand_string()
now = datetime.utcnow().isoformat()
soap_action = rand_string()
requests_module = _FakeRequestsModule()
config = self._get_config()
config['data_format'] = DATA_FORMAT.XML
config['transport'] = URL_TYPE.SOAP
config['soap_action'] = soap_action
config['soap_version'] = '1.2'
wrapper = HTTPSOAPWrapper(config, requests_module)
user_headers = {rand_string():rand_string(), rand_string():rand_string()}
headers = wrapper._create_headers(cid, user_headers, now)
eq_(headers.pop('X-Zato-CID'), cid)
eq_(headers.pop('X-Zato-Msg-TS'), now)
eq_(headers.pop('X-Zato-Component'), get_component_name())
eq_(headers.pop('SOAPAction'), soap_action)
eq_(headers.pop('Content-Type'), CONTENT_TYPE.SOAP12)
# Anything that is left must be user headers
self.assertDictEqual(headers, user_headers)
def __init__(self, config, _requests_session=None):
self.config = config
self.config['timeout'] = float(
self.config['timeout']) if self.config['timeout'] else 0
self.config_no_sensitive = deepcopy(self.config)
self.config_no_sensitive['password'] = '******'
self.requests_session = requests_session or _requests_session
self.session = requests_session(pool_maxsize=self.config['pool_size'])
self.https_adapter = HTTPSAdapter()
self.session.mount('https://', self.https_adapter)
self._component_name = get_component_name()
self.default_content_type = self.get_default_content_type()
self.address = None
self.path_params = []
self.base_headers = {}
# API keys
if self.config['sec_type'] == SEC_DEF_TYPE.APIKEY:
username = self.config.get('orig_username')
if not username:
username = self.config['username']
self.base_headers[username] = self.config['password']
self.set_address_data()
def test_create_headers_soap12(self):
cid = rand_string()
now = datetime.utcnow().isoformat()
soap_action = rand_string()
requests_module = _FakeRequestsModule()
config = self._get_config()
config['data_format'] = DATA_FORMAT.XML
config['transport'] = URL_TYPE.SOAP
config['soap_action'] = soap_action
config['soap_version'] = '1.2'
wrapper = HTTPSOAPWrapper(config, requests_module)
user_headers = {rand_string():rand_string(), rand_string():rand_string()}
headers = wrapper._create_headers(cid, user_headers, now)
eq_(headers.pop('X-Zato-CID'), cid)
eq_(headers.pop('X-Zato-Msg-TS'), now)
eq_(headers.pop('X-Zato-Component'), get_component_name())
eq_(headers.pop('SOAPAction'), soap_action)
eq_(headers.pop('Content-Type'), CONTENT_TYPE.SOAP12)
# Anything that is left must be user headers
self.assertDictEqual(headers, user_headers)
def __init__(self, config, requests_module=None):
self.config = config
self.config_no_sensitive = deepcopy(self.config)
self.config_no_sensitive['password'] = '******'
self.requests_module = requests_module or requests
self.session = self.requests_module.session(
pool_maxsize=self.config['pool_size'])
self._component_name = get_component_name()
self.soap = {}
self.soap['1.1'] = {}
self.soap['1.1']['content_type'] = 'text/xml; charset=utf-8'
self.soap['1.1']['message'] = """<?xml version="1.0" encoding="utf-8"?>
<s11:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s11="http://schemas.xmlsoap.org/soap/envelope/">
{header}
<s11:Body>{data}</s11:Body>
</s11:Envelope>"""
self.soap['1.1'][
'header_template'] = """<s11:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" >
<wsse:Security>
<wsse:UsernameToken>
<wsse:Username>{Username}</wsse:Username>
<wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password>
</wsse:UsernameToken>
</wsse:Security>
</s11:Header>
"""
self.soap['1.2'] = {}
self.soap['1.2'][
'content_type'] = 'application/soap+xml; charset=utf-8'
self.soap['1.2']['message'] = """<?xml version="1.0" encoding="utf-8"?>
<s12:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s12="http://www.w3.org/2003/05/soap-envelope/">
{header}
<s12:Body></s12:Body>
</s12:Envelope>"""
self.soap['1.2'][
'header_template'] = """<s12:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" >
<wsse:Security>
<wsse:UsernameToken>
<wsse:Username>{Username}</wsse:Username>
<wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password>
</wsse:UsernameToken>
</wsse:Security>
</s12:Header>
"""
self.address = None
self.path_params = []
self.set_address_data()
self.set_auth()
def __init__(_py_amqp_self, *args, **kwargs):
super(_PyAMQPConnection,
_py_amqp_self).__init__(client_properties={
'zato.component':
'{}/{}'.format(get_component_name('amqp-conn'),
suffix),
'zato.version':
version,
'zato.definition.name':
self.config.name,
},
*args,
**kwargs)
def __init__(self, config, requests_module=None):
self.config = config
self.config_no_sensitive = deepcopy(self.config)
self.config_no_sensitive['password'] = '******'
self.requests_module = requests_module or requests
self.session = self.requests_module.session(pool_maxsize=self.config['pool_size'])
self._component_name = get_component_name()
self.address = None
self.path_params = []
self.set_address_data()
self.set_auth()
def __init__(self, config, requests_module=None):
self.config = config
self.config_no_sensitive = deepcopy(self.config)
self.config_no_sensitive['password'] = '******'
self.requests_module = requests_module or requests
self.session = self.requests_module.session(
pool_maxsize=self.config['pool_size'])
self._component_name = get_component_name()
self.address = None
self.path_params = []
self.set_address_data()
self.set_auth()
def __init__(self, config, requests_module=None):
self.config = config
self.config_no_sensitive = deepcopy(self.config)
self.config_no_sensitive['password'] = '******'
self.requests_module = requests_module or requests
self.session = self.requests_module.session(pool_maxsize=self.config['pool_size'])
self._component_name = get_component_name()
self.soap = {}
self.soap['1.1'] = {}
self.soap['1.1']['content_type'] = 'text/xml; charset=utf-8'
self.soap['1.1']['message'] = """<?xml version="1.0" encoding="utf-8"?>
<s11:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s11="http://schemas.xmlsoap.org/soap/envelope/">
{header}
<s11:Body>{data}</s11:Body>
</s11:Envelope>"""
self.soap['1.1']['header_template'] = """<s11:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" >
<wsse:Security>
<wsse:UsernameToken>
<wsse:Username>{Username}</wsse:Username>
<wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password>
</wsse:UsernameToken>
</wsse:Security>
</s11:Header>
"""
self.soap['1.2'] = {}
self.soap['1.2']['content_type'] = 'application/soap+xml; charset=utf-8'
self.soap['1.2']['message'] = """<?xml version="1.0" encoding="utf-8"?>
<s12:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s12="http://www.w3.org/2003/05/soap-envelope/">
{header}
<s12:Body></s12:Body>
</s12:Envelope>"""
self.soap['1.2']['header_template'] = """<s12:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" >
<wsse:Security>
<wsse:UsernameToken>
<wsse:Username>{Username}</wsse:Username>
<wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password>
</wsse:UsernameToken>
</wsse:Security>
</s12:Header>
"""
self.address = None
self.path_params = []
self.set_address_data()
self.set_auth()
def __init__(self, config, requests_module=None):
self.config = config
self.config['timeout'] = float(self.config['timeout'])
self.config_no_sensitive = deepcopy(self.config)
self.config_no_sensitive['password'] = '******'
self.requests_module = requests_module or requests
self.session = self.requests_module.session(pool_maxsize=self.config['pool_size'])
self.https_adapter = HTTPSAdapter()
self.session.mount('https://', self.https_adapter)
self._component_name = get_component_name()
self.default_content_type = self.get_default_content_type()
self.address = None
self.path_params = []
self.set_address_data()
def __init__(self, config, requests_module=None):
self.config = config
self.config['timeout'] = float(self.config['timeout'])
self.config_no_sensitive = deepcopy(self.config)
self.config_no_sensitive['password'] = '******'
self.requests_module = requests_module or requests
self.session = self.requests_module.session(pool_maxsize=self.config['pool_size'])
self.https_adapter = HTTPSAdapter()
self.session.mount('https://', self.https_adapter)
self._component_name = get_component_name()
self.address = None
self.path_params = []
self.set_address_data()
self.set_auth()
def __init__(self, repo_location=None, def_id=None, out_id=None, init=True):
super(OutgoingConnector, self).__init__(repo_location, def_id)
self.out_id = out_id
self.broker_client_id = 'amqp-publishing-connector'
self.broker_callbacks = {
TOPICS[MESSAGE_TYPE.TO_AMQP_PUBLISHING_CONNECTOR_ALL]: self.on_broker_msg,
TOPICS[MESSAGE_TYPE.TO_AMQP_CONNECTOR_ALL]: self.on_broker_msg
}
self.broker_messages = self.broker_callbacks.keys()
self.component_name = get_component_name('out-amqp')
if init:
self._init()
logger.info('Started an AMQP publisher for [{}]'.format(self._conn_info()))
def _get_consumer(self, _no_ack=no_ack, _gevent_sleep=sleep):
""" Creates a new connection and consumer to an AMQP broker.
"""
# We cannot assume that we will obtain the consumer right-away. For instance, the remote end
# may be currently available when we are starting. It's OK to block indefinitely (or until self.keep_running is False)
# because we run in our own greenlet.
consumer = None
err_conn_attempts = 0
while not consumer:
if not self.keep_running:
break
try:
conn = self.config.conn_class(self.config.conn_url)
consumer = _Consumer(conn,
queues=self.queue,
callbacks=[self._on_amqp_message],
no_ack=_no_ack[self.config.ack_mode],
tag_prefix='{}/{}'.format(
self.config.consumer_tag_prefix,
get_component_name('amqp-consumer')))
consumer.qos(prefetch_size=0,
prefetch_count=self.config.prefetch_count,
apply_global=False)
consumer.consume()
except Exception:
err_conn_attempts += 1
noun = 'attempts' if err_conn_attempts > 1 else 'attempt'
logger.info(
'Could not create an AMQP consumer for channel `%s` (%s %s so far), e:`%s`',
self.name, err_conn_attempts, noun, format_exc())
# It's fine to sleep for a longer time because if this exception happens it means that we cannot connect
# to the server at all, which will likely mean that it is down,
if self.keep_running:
_gevent_sleep(2)
if err_conn_attempts > 0:
noun = 'attempts' if err_conn_attempts > 1 else 'attempt'
logger.info(
'Created an AMQP consumer for channel `%s` after %s %s',
self.name, err_conn_attempts, noun)
return consumer
def test_create_headers_no_data_format(self):
cid = rand_string()
now = datetime.utcnow().isoformat()
requests_module = _FakeRequestsModule()
config = self._get_config()
wrapper = HTTPSOAPWrapper(config, requests_module)
user_headers = {rand_string():rand_string(), rand_string():rand_string()}
headers = wrapper._create_headers(cid, user_headers, now)
eq_(headers.pop('X-Zato-CID'), cid)
eq_(headers.pop('X-Zato-Msg-TS'), now)
eq_(headers.pop('X-Zato-Component'), get_component_name())
# Anything that is left must be user headers
# (note that there was no Content-Type because there was no data_format)
self.assertDictEqual(headers, user_headers)
def test_create_headers_no_data_format(self):
cid = rand_string()
now = datetime.utcnow().isoformat()
requests_module = _FakeRequestsModule()
config = self._get_config(False)
wrapper = HTTPSOAPWrapper(config, requests_module)
user_headers = {rand_string():rand_string(), rand_string():rand_string()}
headers = wrapper._create_headers(cid, user_headers, now)
eq_(headers.pop('X-Zato-CID'), cid)
eq_(headers.pop('X-Zato-Msg-TS'), now)
eq_(headers.pop('X-Zato-Component'), get_component_name())
# Anything that is left must be user headers
# (note that there was no Content-Type because there was no data_format)
self.assertDictEqual(headers, user_headers)
def establish_connection(self):
conninfo = self.client
for name, default_value in self.default_connection_params.items():
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.hostname == 'localhost':
conninfo.hostname = '127.0.0.1'
conn = self.Connection(host=conninfo.host,
userid=conninfo.userid,
password=conninfo.password,
login_method=conninfo.login_method,
virtual_host=conninfo.virtual_host,
insist=conninfo.insist,
ssl=conninfo.ssl,
connect_timeout=conninfo.connect_timeout,
heartbeat=conninfo.heartbeat,
client_properties={'zato-component':get_component_name(COMPONENT_PREFIX)})
conn.client = self.client
return conn
def __init__(self, name, config, config_no_sensitive):
self.name = name
self.config = config
self.logger = getLogger(self.__class__.__name__)
# Safe for printing out to logs, any sensitive data has been shadowed
self.config_no_sensitive = config_no_sensitive
_extra = {}
_extra['connect_args'] = {'application_name': get_component_name()}
extra = self.config.get('extra') # Optional, hence .get
_extra.update(parse_extra_into_dict(extra))
engine_url = engine_def.format(**config)
self.engine = create_engine(engine_url, pool_size=int(config['pool_size']), **_extra)
event.listen(self.engine, 'checkin', self.on_checkin)
event.listen(self.engine, 'checkout', self.on_checkout)
event.listen(self.engine, 'connect', self.on_connect)
event.listen(self.engine, 'first_connect', self.on_first_connect)
def test_create_headers_user_default_data_format(self):
cid = rand_string()
now = datetime.utcnow().isoformat()
content_type = rand_string()
requests_module = _FakeRequestsModule()
config = self._get_config()
config['content_type'] = content_type
wrapper = HTTPSOAPWrapper(config, requests_module)
user_headers = {rand_string():rand_string(), rand_string():rand_string()}
headers = wrapper._create_headers(cid, user_headers, now)
eq_(headers.pop('X-Zato-CID'), cid)
eq_(headers.pop('X-Zato-Msg-TS'), now)
eq_(headers.pop('X-Zato-Component'), get_component_name())
eq_(headers.pop('Content-Type'), content_type)
# Anything that is left must be user headers
self.assertDictEqual(headers, user_headers)
def test_create_headers_json(self):
cid = rand_string()
now = datetime.utcnow().isoformat()
requests_module = _FakeRequestsModule()
config = self._get_config()
config['data_format'] = DATA_FORMAT.JSON
config['transport'] = URL_TYPE.PLAIN_HTTP
wrapper = HTTPSOAPWrapper(config, requests_module)
user_headers = {rand_string():rand_string(), rand_string():rand_string()}
headers = wrapper._create_headers(cid, user_headers, now)
eq_(headers.pop('X-Zato-CID'), cid)
eq_(headers.pop('X-Zato-Msg-TS'), now)
eq_(headers.pop('X-Zato-Component'), get_component_name())
eq_(headers.pop('Content-Type'), CONTENT_TYPE.JSON)
# Anything that is left must be user headers
self.assertDictEqual(headers, user_headers)
def __init__(self, name, config, config_no_sensitive):
self.logger = getLogger(self.__class__.__name__)
self.name = name
self.config = config
self.engine_name = config[
'engine'] # self.engine.name is 'mysql' while 'self.engine_name' is mysql+pymysql
# Safe for printing out to logs, any sensitive data has been shadowed
self.config_no_sensitive = config_no_sensitive
_extra = {}
# MySQL only
if self.engine_name.startswith('mysql'):
_extra['pool_recycle'] = 600
# Postgres-only
elif self.engine_name.startswith('postgres'):
_extra['connect_args'] = {'application_name': get_component_name()}
extra = self.config.get('extra') # Optional, hence .get
_extra.update(parse_extra_into_dict(extra))
# SQLite has no pools
if self.engine_name != 'sqlite':
_extra['pool_size'] = int(config.get('pool_size', 1))
if _extra['pool_size'] == 0:
_extra['poolclass'] = NullPool
engine_url = get_engine_url(config)
self.engine = create_engine(engine_url, **_extra)
event.listen(self.engine, 'checkin', self.on_checkin)
event.listen(self.engine, 'checkout', self.on_checkout)
event.listen(self.engine, 'connect', self.on_connect)
event.listen(self.engine, 'first_connect', self.on_first_connect)
def __init__(self, name, config, config_no_sensitive):
self.logger = getLogger(self.__class__.__name__)
self.name = name
self.config = config
self.engine_name = config["engine"] # self.engine.name is 'mysql' while 'self.engine_name' is mysql+pymysql
# Safe for printing out to logs, any sensitive data has been shadowed
self.config_no_sensitive = config_no_sensitive
_extra = {}
# MySQL only
if self.engine_name.startswith("mysql"):
_extra["pool_recycle"] = 600
# Postgres-only
elif self.engine_name.startswith("postgres"):
_extra["connect_args"] = {"application_name": get_component_name()}
extra = self.config.get("extra") # Optional, hence .get
_extra.update(parse_extra_into_dict(extra))
# SQLite has no pools
if self.engine_name != "sqlite":
_extra["pool_size"] = int(config.get("pool_size", 1))
if _extra["pool_size"] == 0:
_extra["poolclass"] = NullPool
engine_url = get_engine_url(config)
self.engine = create_engine(engine_url, **_extra)
event.listen(self.engine, "checkin", self.on_checkin)
event.listen(self.engine, "checkout", self.on_checkout)
event.listen(self.engine, "connect", self.on_connect)
event.listen(self.engine, "first_connect", self.on_first_connect)
def _get_engine(self, args):
connect_args = {
'application_name': util.get_component_name('enmasse')
} if args.odb_type == 'postgresql' else {}
return sqlalchemy.create_engine(odb_util.get_engine_url(args),
connect_args=connect_args)
def _get_engine(self, args):
connect_args = {'application_name':util.get_component_name('enmasse')} if args.odb_type == 'postgresql' else {}
return sqlalchemy.create_engine(odb_util.get_engine_url(args), connect_args=connect_args)
def _get_engine(self, args):
connect_args = {"application_name": util.get_component_name("enmasse")} if args.odb_type == "postgresql" else {}
return sqlalchemy.create_engine(get_engine_url(args), connect_args=connect_args)
