def __init__(self, name, config, config_no_sensitive): self.logger = getLogger(self.__class__.__name__) self.name = name self.config = config self.engine_name = config['engine'] # self.engine.name is 'mysql' while 'self.engine_name' is mysql+pymysql # Safe for printing out to logs, any sensitive data has been shadowed self.config_no_sensitive = config_no_sensitive _extra = {} # Postgres-only if self.engine_name.startswith('mysql'): _extra['pool_recycle'] = 600 elif self.engine_name.startswith('postgres'): _extra['connect_args'] = {'application_name': get_component_name()} extra = self.config.get('extra') # Optional, hence .get _extra.update(parse_extra_into_dict(extra)) engine_url = engine_def.format(**config) self.engine = create_engine(engine_url, pool_size=int(config['pool_size']), **_extra) event.listen(self.engine, 'checkin', self.on_checkin) event.listen(self.engine, 'checkout', self.on_checkout) event.listen(self.engine, 'connect', self.on_connect) event.listen(self.engine, 'first_connect', self.on_first_connect)
def _get_engine(self, args): engine_url = odb.engine_def.format(engine=args.odb_type, username=args.odb_user, password=args.odb_password, host=args.odb_host, port=args.odb_port, db_name=args.odb_db_name) connect_args = {'application_name':util.get_component_name('enmasse')} if args.odb_type == 'postgresql' else {} return sqlalchemy.create_engine(engine_url, connect_args=connect_args)
def test_create_headers_json(self): cid = rand_string() now = datetime.utcnow().isoformat() requests_module = _FakeRequestsModule() config = self._get_config() config['data_format'] = DATA_FORMAT.JSON config['transport'] = URL_TYPE.PLAIN_HTTP wrapper = HTTPSOAPWrapper(config, requests_module) user_headers = { rand_string(): rand_string(), rand_string(): rand_string() } headers = wrapper._create_headers(cid, user_headers, now) eq_(headers.pop('X-Zato-CID'), cid) eq_(headers.pop('X-Zato-Msg-TS'), now) eq_(headers.pop('X-Zato-Component'), get_component_name()) eq_(headers.pop('Content-Type'), CONTENT_TYPE.JSON) # Anything that is left must be user headers self.assertDictEqual(headers, user_headers)
def test_create_headers_soap12(self): cid = rand_string() now = datetime.utcnow().isoformat() soap_action = rand_string() requests_module = _FakeRequestsModule() config = self._get_config() config['data_format'] = DATA_FORMAT.XML config['transport'] = URL_TYPE.SOAP config['soap_action'] = soap_action config['soap_version'] = '1.2' wrapper = HTTPSOAPWrapper(config, requests_module) user_headers = {rand_string():rand_string(), rand_string():rand_string()} headers = wrapper._create_headers(cid, user_headers, now) eq_(headers.pop('X-Zato-CID'), cid) eq_(headers.pop('X-Zato-Msg-TS'), now) eq_(headers.pop('X-Zato-Component'), get_component_name()) eq_(headers.pop('SOAPAction'), soap_action) eq_(headers.pop('Content-Type'), CONTENT_TYPE.SOAP12) # Anything that is left must be user headers self.assertDictEqual(headers, user_headers)
def __init__(self, config, _requests_session=None): self.config = config self.config['timeout'] = float( self.config['timeout']) if self.config['timeout'] else 0 self.config_no_sensitive = deepcopy(self.config) self.config_no_sensitive['password'] = '******' self.requests_session = requests_session or _requests_session self.session = requests_session(pool_maxsize=self.config['pool_size']) self.https_adapter = HTTPSAdapter() self.session.mount('https://', self.https_adapter) self._component_name = get_component_name() self.default_content_type = self.get_default_content_type() self.address = None self.path_params = [] self.base_headers = {} # API keys if self.config['sec_type'] == SEC_DEF_TYPE.APIKEY: username = self.config.get('orig_username') if not username: username = self.config['username'] self.base_headers[username] = self.config['password'] self.set_address_data()
def __init__(self, config, requests_module=None): self.config = config self.config_no_sensitive = deepcopy(self.config) self.config_no_sensitive['password'] = '******' self.requests_module = requests_module or requests self.session = self.requests_module.session( pool_maxsize=self.config['pool_size']) self._component_name = get_component_name() self.soap = {} self.soap['1.1'] = {} self.soap['1.1']['content_type'] = 'text/xml; charset=utf-8' self.soap['1.1']['message'] = """<?xml version="1.0" encoding="utf-8"?> <s11:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s11="http://schemas.xmlsoap.org/soap/envelope/"> {header} <s11:Body>{data}</s11:Body> </s11:Envelope>""" self.soap['1.1'][ 'header_template'] = """<s11:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" > <wsse:Security> <wsse:UsernameToken> <wsse:Username>{Username}</wsse:Username> <wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password> </wsse:UsernameToken> </wsse:Security> </s11:Header> """ self.soap['1.2'] = {} self.soap['1.2'][ 'content_type'] = 'application/soap+xml; charset=utf-8' self.soap['1.2']['message'] = """<?xml version="1.0" encoding="utf-8"?> <s12:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s12="http://www.w3.org/2003/05/soap-envelope/"> {header} <s12:Body></s12:Body> </s12:Envelope>""" self.soap['1.2'][ 'header_template'] = """<s12:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" > <wsse:Security> <wsse:UsernameToken> <wsse:Username>{Username}</wsse:Username> <wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password> </wsse:UsernameToken> </wsse:Security> </s12:Header> """ self.address = None self.path_params = [] self.set_address_data() self.set_auth()
def __init__(_py_amqp_self, *args, **kwargs): super(_PyAMQPConnection, _py_amqp_self).__init__(client_properties={ 'zato.component': '{}/{}'.format(get_component_name('amqp-conn'), suffix), 'zato.version': version, 'zato.definition.name': self.config.name, }, *args, **kwargs)
def __init__(self, config, requests_module=None): self.config = config self.config_no_sensitive = deepcopy(self.config) self.config_no_sensitive['password'] = '******' self.requests_module = requests_module or requests self.session = self.requests_module.session(pool_maxsize=self.config['pool_size']) self._component_name = get_component_name() self.address = None self.path_params = [] self.set_address_data() self.set_auth()
def __init__(self, config, requests_module=None): self.config = config self.config_no_sensitive = deepcopy(self.config) self.config_no_sensitive['password'] = '******' self.requests_module = requests_module or requests self.session = self.requests_module.session( pool_maxsize=self.config['pool_size']) self._component_name = get_component_name() self.address = None self.path_params = [] self.set_address_data() self.set_auth()
def __init__(self, config, requests_module=None): self.config = config self.config_no_sensitive = deepcopy(self.config) self.config_no_sensitive['password'] = '******' self.requests_module = requests_module or requests self.session = self.requests_module.session(pool_maxsize=self.config['pool_size']) self._component_name = get_component_name() self.soap = {} self.soap['1.1'] = {} self.soap['1.1']['content_type'] = 'text/xml; charset=utf-8' self.soap['1.1']['message'] = """<?xml version="1.0" encoding="utf-8"?> <s11:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s11="http://schemas.xmlsoap.org/soap/envelope/"> {header} <s11:Body>{data}</s11:Body> </s11:Envelope>""" self.soap['1.1']['header_template'] = """<s11:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" > <wsse:Security> <wsse:UsernameToken> <wsse:Username>{Username}</wsse:Username> <wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password> </wsse:UsernameToken> </wsse:Security> </s11:Header> """ self.soap['1.2'] = {} self.soap['1.2']['content_type'] = 'application/soap+xml; charset=utf-8' self.soap['1.2']['message'] = """<?xml version="1.0" encoding="utf-8"?> <s12:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:s12="http://www.w3.org/2003/05/soap-envelope/"> {header} <s12:Body></s12:Body> </s12:Envelope>""" self.soap['1.2']['header_template'] = """<s12:Header xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" > <wsse:Security> <wsse:UsernameToken> <wsse:Username>{Username}</wsse:Username> <wsse:Password Type="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText">{Password}</wsse:Password> </wsse:UsernameToken> </wsse:Security> </s12:Header> """ self.address = None self.path_params = [] self.set_address_data() self.set_auth()
def __init__(self, config, requests_module=None): self.config = config self.config['timeout'] = float(self.config['timeout']) self.config_no_sensitive = deepcopy(self.config) self.config_no_sensitive['password'] = '******' self.requests_module = requests_module or requests self.session = self.requests_module.session(pool_maxsize=self.config['pool_size']) self.https_adapter = HTTPSAdapter() self.session.mount('https://', self.https_adapter) self._component_name = get_component_name() self.default_content_type = self.get_default_content_type() self.address = None self.path_params = [] self.set_address_data()
def __init__(self, config, requests_module=None): self.config = config self.config['timeout'] = float(self.config['timeout']) self.config_no_sensitive = deepcopy(self.config) self.config_no_sensitive['password'] = '******' self.requests_module = requests_module or requests self.session = self.requests_module.session(pool_maxsize=self.config['pool_size']) self.https_adapter = HTTPSAdapter() self.session.mount('https://', self.https_adapter) self._component_name = get_component_name() self.address = None self.path_params = [] self.set_address_data() self.set_auth()
def __init__(self, repo_location=None, def_id=None, out_id=None, init=True): super(OutgoingConnector, self).__init__(repo_location, def_id) self.out_id = out_id self.broker_client_id = 'amqp-publishing-connector' self.broker_callbacks = { TOPICS[MESSAGE_TYPE.TO_AMQP_PUBLISHING_CONNECTOR_ALL]: self.on_broker_msg, TOPICS[MESSAGE_TYPE.TO_AMQP_CONNECTOR_ALL]: self.on_broker_msg } self.broker_messages = self.broker_callbacks.keys() self.component_name = get_component_name('out-amqp') if init: self._init() logger.info('Started an AMQP publisher for [{}]'.format(self._conn_info()))
def _get_consumer(self, _no_ack=no_ack, _gevent_sleep=sleep): """ Creates a new connection and consumer to an AMQP broker. """ # We cannot assume that we will obtain the consumer right-away. For instance, the remote end # may be currently available when we are starting. It's OK to block indefinitely (or until self.keep_running is False) # because we run in our own greenlet. consumer = None err_conn_attempts = 0 while not consumer: if not self.keep_running: break try: conn = self.config.conn_class(self.config.conn_url) consumer = _Consumer(conn, queues=self.queue, callbacks=[self._on_amqp_message], no_ack=_no_ack[self.config.ack_mode], tag_prefix='{}/{}'.format( self.config.consumer_tag_prefix, get_component_name('amqp-consumer'))) consumer.qos(prefetch_size=0, prefetch_count=self.config.prefetch_count, apply_global=False) consumer.consume() except Exception: err_conn_attempts += 1 noun = 'attempts' if err_conn_attempts > 1 else 'attempt' logger.info( 'Could not create an AMQP consumer for channel `%s` (%s %s so far), e:`%s`', self.name, err_conn_attempts, noun, format_exc()) # It's fine to sleep for a longer time because if this exception happens it means that we cannot connect # to the server at all, which will likely mean that it is down, if self.keep_running: _gevent_sleep(2) if err_conn_attempts > 0: noun = 'attempts' if err_conn_attempts > 1 else 'attempt' logger.info( 'Created an AMQP consumer for channel `%s` after %s %s', self.name, err_conn_attempts, noun) return consumer
def test_create_headers_no_data_format(self): cid = rand_string() now = datetime.utcnow().isoformat() requests_module = _FakeRequestsModule() config = self._get_config() wrapper = HTTPSOAPWrapper(config, requests_module) user_headers = {rand_string():rand_string(), rand_string():rand_string()} headers = wrapper._create_headers(cid, user_headers, now) eq_(headers.pop('X-Zato-CID'), cid) eq_(headers.pop('X-Zato-Msg-TS'), now) eq_(headers.pop('X-Zato-Component'), get_component_name()) # Anything that is left must be user headers # (note that there was no Content-Type because there was no data_format) self.assertDictEqual(headers, user_headers)
def test_create_headers_no_data_format(self): cid = rand_string() now = datetime.utcnow().isoformat() requests_module = _FakeRequestsModule() config = self._get_config(False) wrapper = HTTPSOAPWrapper(config, requests_module) user_headers = {rand_string():rand_string(), rand_string():rand_string()} headers = wrapper._create_headers(cid, user_headers, now) eq_(headers.pop('X-Zato-CID'), cid) eq_(headers.pop('X-Zato-Msg-TS'), now) eq_(headers.pop('X-Zato-Component'), get_component_name()) # Anything that is left must be user headers # (note that there was no Content-Type because there was no data_format) self.assertDictEqual(headers, user_headers)
def establish_connection(self): conninfo = self.client for name, default_value in self.default_connection_params.items(): if not getattr(conninfo, name, None): setattr(conninfo, name, default_value) if conninfo.hostname == 'localhost': conninfo.hostname = '127.0.0.1' conn = self.Connection(host=conninfo.host, userid=conninfo.userid, password=conninfo.password, login_method=conninfo.login_method, virtual_host=conninfo.virtual_host, insist=conninfo.insist, ssl=conninfo.ssl, connect_timeout=conninfo.connect_timeout, heartbeat=conninfo.heartbeat, client_properties={'zato-component':get_component_name(COMPONENT_PREFIX)}) conn.client = self.client return conn
def __init__(self, name, config, config_no_sensitive): self.name = name self.config = config self.logger = getLogger(self.__class__.__name__) # Safe for printing out to logs, any sensitive data has been shadowed self.config_no_sensitive = config_no_sensitive _extra = {} _extra['connect_args'] = {'application_name': get_component_name()} extra = self.config.get('extra') # Optional, hence .get _extra.update(parse_extra_into_dict(extra)) engine_url = engine_def.format(**config) self.engine = create_engine(engine_url, pool_size=int(config['pool_size']), **_extra) event.listen(self.engine, 'checkin', self.on_checkin) event.listen(self.engine, 'checkout', self.on_checkout) event.listen(self.engine, 'connect', self.on_connect) event.listen(self.engine, 'first_connect', self.on_first_connect)
def test_create_headers_user_default_data_format(self): cid = rand_string() now = datetime.utcnow().isoformat() content_type = rand_string() requests_module = _FakeRequestsModule() config = self._get_config() config['content_type'] = content_type wrapper = HTTPSOAPWrapper(config, requests_module) user_headers = {rand_string():rand_string(), rand_string():rand_string()} headers = wrapper._create_headers(cid, user_headers, now) eq_(headers.pop('X-Zato-CID'), cid) eq_(headers.pop('X-Zato-Msg-TS'), now) eq_(headers.pop('X-Zato-Component'), get_component_name()) eq_(headers.pop('Content-Type'), content_type) # Anything that is left must be user headers self.assertDictEqual(headers, user_headers)
def test_create_headers_json(self): cid = rand_string() now = datetime.utcnow().isoformat() requests_module = _FakeRequestsModule() config = self._get_config() config['data_format'] = DATA_FORMAT.JSON config['transport'] = URL_TYPE.PLAIN_HTTP wrapper = HTTPSOAPWrapper(config, requests_module) user_headers = {rand_string():rand_string(), rand_string():rand_string()} headers = wrapper._create_headers(cid, user_headers, now) eq_(headers.pop('X-Zato-CID'), cid) eq_(headers.pop('X-Zato-Msg-TS'), now) eq_(headers.pop('X-Zato-Component'), get_component_name()) eq_(headers.pop('Content-Type'), CONTENT_TYPE.JSON) # Anything that is left must be user headers self.assertDictEqual(headers, user_headers)
def __init__(self, name, config, config_no_sensitive): self.logger = getLogger(self.__class__.__name__) self.name = name self.config = config self.engine_name = config[ 'engine'] # self.engine.name is 'mysql' while 'self.engine_name' is mysql+pymysql # Safe for printing out to logs, any sensitive data has been shadowed self.config_no_sensitive = config_no_sensitive _extra = {} # MySQL only if self.engine_name.startswith('mysql'): _extra['pool_recycle'] = 600 # Postgres-only elif self.engine_name.startswith('postgres'): _extra['connect_args'] = {'application_name': get_component_name()} extra = self.config.get('extra') # Optional, hence .get _extra.update(parse_extra_into_dict(extra)) # SQLite has no pools if self.engine_name != 'sqlite': _extra['pool_size'] = int(config.get('pool_size', 1)) if _extra['pool_size'] == 0: _extra['poolclass'] = NullPool engine_url = get_engine_url(config) self.engine = create_engine(engine_url, **_extra) event.listen(self.engine, 'checkin', self.on_checkin) event.listen(self.engine, 'checkout', self.on_checkout) event.listen(self.engine, 'connect', self.on_connect) event.listen(self.engine, 'first_connect', self.on_first_connect)
def __init__(self, name, config, config_no_sensitive): self.logger = getLogger(self.__class__.__name__) self.name = name self.config = config self.engine_name = config["engine"] # self.engine.name is 'mysql' while 'self.engine_name' is mysql+pymysql # Safe for printing out to logs, any sensitive data has been shadowed self.config_no_sensitive = config_no_sensitive _extra = {} # MySQL only if self.engine_name.startswith("mysql"): _extra["pool_recycle"] = 600 # Postgres-only elif self.engine_name.startswith("postgres"): _extra["connect_args"] = {"application_name": get_component_name()} extra = self.config.get("extra") # Optional, hence .get _extra.update(parse_extra_into_dict(extra)) # SQLite has no pools if self.engine_name != "sqlite": _extra["pool_size"] = int(config.get("pool_size", 1)) if _extra["pool_size"] == 0: _extra["poolclass"] = NullPool engine_url = get_engine_url(config) self.engine = create_engine(engine_url, **_extra) event.listen(self.engine, "checkin", self.on_checkin) event.listen(self.engine, "checkout", self.on_checkout) event.listen(self.engine, "connect", self.on_connect) event.listen(self.engine, "first_connect", self.on_first_connect)
def _get_engine(self, args): connect_args = { 'application_name': util.get_component_name('enmasse') } if args.odb_type == 'postgresql' else {} return sqlalchemy.create_engine(odb_util.get_engine_url(args), connect_args=connect_args)
def _get_engine(self, args): connect_args = {'application_name':util.get_component_name('enmasse')} if args.odb_type == 'postgresql' else {} return sqlalchemy.create_engine(odb_util.get_engine_url(args), connect_args=connect_args)
def _get_engine(self, args): connect_args = {"application_name": util.get_component_name("enmasse")} if args.odb_type == "postgresql" else {} return sqlalchemy.create_engine(get_engine_url(args), connect_args=connect_args)