def print_async_event(self, suffix, event): ''' Print all of the events with the prefix 'tag' ''' if not isinstance(event, dict): return # if we are "quiet", don't print if self.opts.get('quiet', False): return # some suffixes we don't want to print if suffix in ('new',): return outputter = self.opts.get('output', event.get('outputter', None)) # if this is a ret, we have our own set of rules if suffix == 'ret': # Check if ouputter was passed in the return data. If this is the case, # then the return data will be a dict two keys: 'data' and 'outputter' if isinstance(event.get('return'), dict) \ and set(event['return']) == set(('data', 'outputter')): event_data = event['return']['data'] outputter = event['return']['outputter'] else: event_data = event['return'] else: event_data = {'suffix': suffix, 'event': event} salt.output.display_output(event_data, outputter, self.opts)
def _update_resources(event, options): ''' TODO: document this method ''' conn = _get_conn(options) cur = conn.cursor() cur.execute('SELECT id FROM salt_resources') resources_db = [res[0] for res in cur.fetchall()] resources = event.get('return', {}).values() for resource in resources: rid = '%s|%s' % (event.get('id'), resource.get('__id__')) if rid in resources_db: status = 'unknown' if resource.get('result', None) is not None: status = 'success' if resource.get('result') else 'failed' resource_sql = '''UPDATE salt_resources SET (status, last_ret, alter_time) = (%s, %s, current_timestamp) WHERE id = %s''' cur.execute(resource_sql, (status, repr(resource), rid)) conn.commit() _close_conn(conn)
def print_async_event(self, suffix, event): ''' Print all of the events with the prefix 'tag' ''' if not isinstance(event, dict): return # if we are "quiet", don't print if self.opts.get('quiet', False): return # some suffixes we don't want to print if suffix in ('new',): return try: outputter = self.opts.get('output', event.get('outputter', None) or event.get('return').get('outputter')) except AttributeError: outputter = None # if this is a ret, we have our own set of rules if suffix == 'ret': # Check if outputter was passed in the return data. If this is the case, # then the return data will be a dict two keys: 'data' and 'outputter' if isinstance(event.get('return'), dict) \ and set(event['return']) == set(('data', 'outputter')): event_data = event['return']['data'] outputter = event['return']['outputter'] else: event_data = event['return'] else: event_data = {'suffix': suffix, 'event': event} salt.output.display_output(event_data, outputter, self.opts)
def test_reactor_is_leader(self): """ when leader is set to false reactor should timeout/not do anything """ # by default reactor should be leader ret = self.run_run_plus("reactor.is_leader") self.assertTrue(ret["return"]) # make reactor not leader self.run_run_plus("reactor.set_leader", False) ret = self.run_run_plus("reactor.is_leader") self.assertFalse(ret["return"]) signal.signal(signal.SIGALRM, self.alarm_handler) signal.alarm(self.timeout) try: master_event = self.get_event() self.fire_event({"id": "minion"}, "salt/test/reactor") while True: event = master_event.get_event(full=True) if event is None: continue if event.get("tag") == "test_reaction": # if we reach this point, the test is a failure self.assertTrue(False) # pylint: disable=redundant-unittest-assert break except TimeoutException as exc: self.assertTrue("Timeout" in str(exc)) finally: signal.alarm(0) # make reactor leader again self.run_run_plus("reactor.set_leader", True) ret = self.run_run_plus("reactor.is_leader") self.assertTrue(ret["return"]) # trigger a reaction signal.alarm(self.timeout) try: master_event = self.get_event() self.fire_event({"id": "minion"}, "salt/test/reactor") while True: event = master_event.get_event(full=True) if event is None: continue if event.get("tag") == "test_reaction": self.assertTrue(event["data"]["test_reaction"]) break finally: signal.alarm(0)
def test_reactor_is_leader(self): ''' when leader is set to false reactor should timeout/not do anything ''' # by default reactor should be leader ret = self.run_run_plus('reactor.is_leader') self.assertTrue(ret['return']) # make reactor not leader self.run_run_plus('reactor.set_leader', False) ret = self.run_run_plus('reactor.is_leader') self.assertFalse(ret['return']) signal.signal(signal.SIGALRM, self.alarm_handler) signal.alarm(self.timeout) try: master_event = self.get_event() self.fire_event({'id': 'minion'}, 'salt/test/reactor') while True: event = master_event.get_event(full=True) if event is None: continue if event.get('tag') == 'test_reaction': # if we reach this point, the test is a failure self.assertTrue(False) # pylint: disable=redundant-unittest-assert break except TimeoutException as exc: self.assertTrue('Timeout' in str(exc)) finally: signal.alarm(0) # make reactor leader again self.run_run_plus('reactor.set_leader', True) ret = self.run_run_plus('reactor.is_leader') self.assertTrue(ret['return']) # trigger a reaction signal.alarm(self.timeout) try: master_event = self.get_event() self.fire_event({'id': 'minion'}, 'salt/test/reactor') while True: event = master_event.get_event(full=True) if event is None: continue if event.get('tag') == 'test_reaction': self.assertTrue(event['data']['test_reaction']) break finally: signal.alarm(0)
def process_event(event): if event is None: return None, None tag = event.get('tag', '') data = event.get('data', {}) if tag == 'salt/event/exit': sys.exit() elif tag in ('salt/auth', 'minion_ping'): return data.get('id'), parse_stamp(data.get('_stamp')) return None, None
def get_finger(self, event=None): """ Get finger from event """ tag = 'other' func = '' if isinstance(event, dict): event_tag = event.get('tag', '') event_data = event.get('data', dict()) for _tag, _rule in self.event_fingers.iteritems(): if _rule.match(event_tag): tag = _tag if _tag.endswith('_job') and event_data: func = event_data.get('fun', '') break return tag, func
def ban_event(event): reason = event['Event'] if event.get('ACLName'): reason = '{} ({})'.format(reason, event.get('ACLName')) account = event['AccountID'] address = event['RemoteAddress'] service = event['Service'] address_parts = address.split('/') log.debug( f'InvalidAccountOrPassword account {service}/{account}, ' f'address {address}') # Check format RemoteAddress='IPV4/UDP/10.18.0.1/35060' if len(address_parts) != 4: log.error(f'Security event parse address error: {event}') return False ip = address_parts[2] return _ban_ip(ip, service=service, account=account, reason=reason)
def start(project='default', host='127.0.0.1', port=8181, username=None, password=None): ''' Listen to state jobs events and forward state functions and node info ''' state_functions = ['state.sls', 'state.apply', 'state.highstate'] model_functions = ['architect.node_info'] class_tag = 'architect/minion/classify' if __opts__['__role'] == 'master': event_bus = salt.utils.event.get_master_event(__opts__, __opts__['sock_dir'], listen=True) else: event_bus = salt.utils.event.get_event('minion', transport=__opts__['transport'], opts=__opts__, sock_dir=__opts__['sock_dir'], listen=True) logger.info('Architect Engine initialised') while True: event = event_bus.get_event() if event and event.get('fun', None) in state_functions: is_test_run = 'test=true' in [ arg.lower() for arg in event.get('fun_args', []) ] if not is_test_run: output = ArchitectClient().push_event(event) logger.info("Sent Architect state function {}".format(output)) if event and event.get('fun', None) in model_functions: output = ArchitectClient().push_node_info( {event['id']: event['return']}) logger.info("Sent Architect node info function {}".format(output)) if event and event.get('tag', None) == class_tag: output = ArchitectClient().classify_node({ 'name': event['id'], 'data': event['data'] }) logger.info("Sent Architect node classification {}".format(output))
def start(host='salt', user='******', password='******', database='salt', port=5432, **kwargs): ''' Listen to events and parse Salt state returns ''' if __opts__['__role'] == 'master': event_bus = salt.utils.event.get_master_event(__opts__, __opts__['sock_dir'], listen=True) else: event_bus = salt.utils.event.get_event('minion', transport=__opts__['transport'], opts=__opts__, sock_dir=__opts__['sock_dir'], listen=True) log.debug('Saltgraph engine started') while True: event = event_bus.get_event() supported_funcs = ['state.sls', 'state.apply', 'state.highstate'] if event and event.get('fun', None) in supported_funcs: test = 'test=true' in [ arg.lower() for arg in event.get('fun_args', []) ] if not test: options = { 'host': host, 'user': user, 'passwd': password, 'db': database, 'port': port } is_reclass = [ arg for arg in event.get('fun_args', []) if arg.startswith('reclass') ] if is_reclass or not _up_to_date(options): _get_lowstate_data(options) _update_resources(event, options)
def print_async_event(self, suffix, event): """ Print all of the events with the prefix 'tag' """ if not isinstance(event, dict): return # if we are "quiet", don't print if self.opts.get("quiet", False): return # some suffixes we don't want to print if suffix in ("new", ): return try: outputter = self.opts.get( "output", event.get("outputter", None) or event.get("return").get("outputter"), ) except AttributeError: outputter = None # if this is a ret, we have our own set of rules if suffix == "ret": # Check if outputter was passed in the return data. If this is the case, # then the return data will be a dict two keys: 'data' and 'outputter' if isinstance(event.get("return"), dict) and set( event["return"]) == { "data", "outputter", }: event_data = event["return"]["data"] outputter = event["return"]["outputter"] else: event_data = event["return"] else: event_data = {"suffix": suffix, "event": event} salt.output.display_output(event_data, outputter, self.opts)
def test_reactor_reaction(self): """ Fire an event on the master and ensure The reactor event responds """ signal.signal(signal.SIGALRM, self.alarm_handler) signal.alarm(self.timeout) master_event = self.get_event() master_event.fire_event({"id": "minion"}, "salt/test/reactor") try: while True: event = master_event.get_event(full=True) if event is None: continue if event.get("tag") == "test_reaction": self.assertTrue(event["data"]["test_reaction"]) break finally: signal.alarm(0)
def get(self): # if you aren't authenticated, redirect to login if not self._verify_auth(): self.redirect('/login') return # set the streaming headers self.set_header('Content-Type', 'text/event-stream') self.set_header('Cache-Control', 'no-cache') self.set_header('Connection', 'keep-alive') self.write(u'retry: {0}\n'.format(400)) self.flush() while True: try: event = yield self.application.event_listener.get_event(self) self.write(u'tag: {0}\n'.format(event.get('tag', ''))) self.write(u'data: {0}\n\n'.format(json.dumps(event))) self.flush() except TimeoutException: break self.finish()
def test_reactor_reaction(self): ''' Fire an event on the master and ensure The reactor event responds ''' signal.signal(signal.SIGALRM, self.alarm_handler) signal.alarm(self.timeout) master_event = self.get_event() master_event.fire_event({'id': 'minion'}, 'salt/test/reactor') try: while True: event = master_event.get_event(full=True) if event is None: continue if event.get('tag') == 'test_reaction': self.assertTrue(event['data']['test_reaction']) break finally: signal.alarm(0)
def test_orchestration_with_pillar_dot_items(self): """ Test to confirm when using a state file that includes other state file, if one of those state files includes pillar related functions that will not be pulling from the pillar cache that all the state files are available and the file_roots has been preserved. See issues #48277 and #46986. """ self.write_conf({ "fileserver_backend": ["roots"], "file_roots": { "base": [self.base_env] } }) orch_sls = os.path.join(self.base_env, "main.sls") with salt.utils.files.fopen(orch_sls, "w") as fp_: fp_.write( textwrap.dedent(""" include: - one - two - three """)) orch_sls = os.path.join(self.base_env, "one.sls") with salt.utils.files.fopen(orch_sls, "w") as fp_: fp_.write( textwrap.dedent(""" {%- set foo = salt['saltutil.runner']('pillar.show_pillar') %} placeholder_one: test.succeed_without_changes """)) orch_sls = os.path.join(self.base_env, "two.sls") with salt.utils.files.fopen(orch_sls, "w") as fp_: fp_.write( textwrap.dedent(""" placeholder_two: test.succeed_without_changes """)) orch_sls = os.path.join(self.base_env, "three.sls") with salt.utils.files.fopen(orch_sls, "w") as fp_: fp_.write( textwrap.dedent(""" placeholder_three: test.succeed_without_changes """)) orch_sls = os.path.join(self.base_env, "main.sls") with salt.utils.event.get_event( "master", sock_dir=self.master_opts["sock_dir"], transport=self.master_opts["transport"], opts=self.master_opts, ) as listener: jid = self.run_run_plus("state.orchestrate", "main").get("jid") if jid is None: raise salt.exceptions.SaltInvocationError( "jid missing from run_run_plus output") signal.signal(signal.SIGALRM, self.alarm_handler) signal.alarm(self.timeout) received = False try: while True: event = listener.get_event(full=True) if event is None: continue if event.get("tag", "") == "salt/run/{}/ret".format(jid): received = True # Don't wrap this in a try/except. We want to know if the # data structure is different from what we expect! ret = event["data"]["return"]["data"]["master"] for state in ret: data = ret[state] # Each state should be successful self.assertEqual(data["comment"], "Success!") break finally: self.assertTrue(received) signal.alarm(0)
def start(cmd, output="json", interval=1): """ Parse stdout of a command and generate an event The script engine will scrap stdout of the given script and generate an event based on the presence of the 'tag' key and its value. If there is a data obj available, that will also be fired along with the tag. Example: Given the following json output from a script: .. code-block:: json { "tag" : "lots/of/tacos", "data" : { "toppings" : "cilantro" } } This will fire the event 'lots/of/tacos' on the event bus with the data obj as is. :param cmd: The command to execute :param output: How to deserialize stdout of the script :param interval: How often to execute the script """ try: cmd = shlex.split(cmd) except AttributeError: cmd = shlex.split(str(cmd)) log.debug("script engine using command %s", cmd) serializer = _get_serializer(output) if __opts__.get("__role") == "master": fire_master = salt.utils.event.get_master_event( __opts__, __opts__["sock_dir"]).fire_event else: fire_master = __salt__["event.send"] while True: try: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) log.debug("Starting script with pid %d", proc.pid) for raw_event in _read_stdout(proc): log.debug(raw_event) event = serializer.deserialize(raw_event) tag = event.get("tag", None) data = event.get("data", {}) if data and "id" not in data: data["id"] = __opts__["id"] if tag: log.info("script engine firing event with tag %s", tag) fire_master(tag=tag, data=data) log.debug("Closing script with pid %d", proc.pid) proc.stdout.close() rc = proc.wait() if rc: raise subprocess.CalledProcessError(rc, cmd) except subprocess.CalledProcessError as e: log.error(e) finally: if proc.poll is None: proc.terminate() time.sleep(interval)
def test_orchestration_with_pillar_dot_items(self): ''' Test to confirm when using a state file that includes other state file, if one of those state files includes pillar related functions that will not be pulling from the pillar cache that all the state files are available and the file_roots has been preserved. See issues #48277 and #46986. ''' self.write_conf({ 'fileserver_backend': ['roots'], 'file_roots': { 'base': [self.base_env], }, }) orch_sls = os.path.join(self.base_env, 'main.sls') with salt.utils.files.fopen(orch_sls, 'w') as fp_: fp_.write( textwrap.dedent(''' include: - one - two - three ''')) orch_sls = os.path.join(self.base_env, 'one.sls') with salt.utils.files.fopen(orch_sls, 'w') as fp_: fp_.write( textwrap.dedent(''' {%- set foo = salt['saltutil.runner']('pillar.show_pillar') %} placeholder_one: test.succeed_without_changes ''')) orch_sls = os.path.join(self.base_env, 'two.sls') with salt.utils.files.fopen(orch_sls, 'w') as fp_: fp_.write( textwrap.dedent(''' placeholder_two: test.succeed_without_changes ''')) orch_sls = os.path.join(self.base_env, 'three.sls') with salt.utils.files.fopen(orch_sls, 'w') as fp_: fp_.write( textwrap.dedent(''' placeholder_three: test.succeed_without_changes ''')) orch_sls = os.path.join(self.base_env, 'main.sls') listener = salt.utils.event.get_event( 'master', sock_dir=self.master_opts['sock_dir'], transport=self.master_opts['transport'], opts=self.master_opts) jid = self.run_run_plus('state.orchestrate', 'main', __reload_config=True).get('jid') if jid is None: raise salt.exceptions.SaltInvocationError( 'jid missing from run_run_plus output') signal.signal(signal.SIGALRM, self.alarm_handler) signal.alarm(self.timeout) received = False try: while True: event = listener.get_event(full=True) if event is None: continue if event.get('tag', '') == 'salt/run/{0}/ret'.format(jid): received = True # Don't wrap this in a try/except. We want to know if the # data structure is different from what we expect! ret = event['data']['return']['data']['master'] for state in ret: data = ret[state] # Each state should be successful self.assertEqual(data['comment'], 'Success!') break finally: self.assertTrue(received) del listener signal.alarm(0)
def get(self): r''' An HTTP stream of the Salt master event bus This stream is formatted per the Server Sent Events (SSE) spec. Each event is formatted as JSON. .. http:get:: /events :status 200: |200| :status 401: |401| :status 406: |406| **Example request:** .. code-block:: bash curl -NsS localhost:8000/events .. code-block:: http GET /events HTTP/1.1 Host: localhost:8000 **Example response:** .. code-block:: http HTTP/1.1 200 OK Connection: keep-alive Cache-Control: no-cache Content-Type: text/event-stream;charset=utf-8 retry: 400 data: {'tag': '', 'data': {'minions': ['ms-4', 'ms-3', 'ms-2', 'ms-1', 'ms-0']}} data: {'tag': '20130802115730568475', 'data': {'jid': '20130802115730568475', 'return': True, 'retcode': 0, 'success': True, 'cmd': '_return', 'fun': 'test.ping', 'id': 'ms-1'}} The event stream can be easily consumed via JavaScript: .. code-block:: javascript # Note, you must be authenticated! var source = new EventSource('/events'); source.onopen = function() { console.debug('opening') }; source.onerror = function(e) { console.debug('error!', e) }; source.onmessage = function(e) { console.debug(e.data) }; Or using CORS: .. code-block:: javascript var source = new EventSource('/events', {withCredentials: true}); Some browser clients lack CORS support for the ``EventSource()`` API. Such clients may instead pass the :mailheader:`X-Auth-Token` value as an URL parameter: .. code-block:: bash curl -NsS localhost:8000/events/6d1b722e It is also possible to consume the stream via the shell. Records are separated by blank lines; the ``data:`` and ``tag:`` prefixes will need to be removed manually before attempting to unserialize the JSON. curl's ``-N`` flag turns off input buffering which is required to process the stream incrementally. Here is a basic example of printing each event as it comes in: .. code-block:: bash curl -NsS localhost:8000/events |\ while IFS= read -r line ; do echo $line done Here is an example of using awk to filter events based on tag: .. code-block:: bash curl -NsS localhost:8000/events |\ awk ' BEGIN { RS=""; FS="\\n" } $1 ~ /^tag: salt\/job\/[0-9]+\/new$/ { print $0 } ' tag: salt/job/20140112010149808995/new data: {"tag": "salt/job/20140112010149808995/new", "data": {"tgt_type": "glob", "jid": "20140112010149808995", "tgt": "jerry", "_stamp": "2014-01-12_01:01:49.809617", "user": "******", "arg": [], "fun": "test.ping", "minions": ["jerry"]}} tag: 20140112010149808995 data: {"tag": "20140112010149808995", "data": {"fun_args": [], "jid": "20140112010149808995", "return": true, "retcode": 0, "success": true, "cmd": "_return", "_stamp": "2014-01-12_01:01:49.819316", "fun": "test.ping", "id": "jerry"}} ''' # if you aren't authenticated, redirect to login if not self._verify_auth(): self.redirect('/login') return # set the streaming headers self.set_header('Content-Type', 'text/event-stream') self.set_header('Cache-Control', 'no-cache') self.set_header('Connection', 'keep-alive') self.write(u'retry: {0}\n'.format(400)) self.flush() while True: try: event = yield self.application.event_listener.get_event(self) self.write(u'tag: {0}\n'.format(event.get('tag', ''))) self.write(u'data: {0}\n\n'.format(json.dumps(event))) self.flush() except TimeoutException: break self.finish()
def test_reactor_is_leader(self): """ If reactor system is unavailable, an exception is thrown. When leader is true (the default), the reacion event should return. When leader is set to false reactor should timeout/not do anything. """ ret = self.run_run_plus("reactor.is_leader") self.assertIn("CommandExecutionError", ret["return"]) self.run_run_plus("reactor.set_leader", False) self.assertIn("CommandExecutionError", ret["return"]) ret = self.run_run_plus("reactor.is_leader") self.assertIn("CommandExecutionError", ret["return"]) # by default reactor should be leader signal.signal(signal.SIGALRM, self.alarm_handler) signal.alarm(self.timeout) # make reactor not the leader # ensure reactor engine is available opts_overrides = { "engines": [{ "reactor": { "refresh_interval": 60, "worker_threads": 10, "worker_hwm": 10000, } }] } self.run_run_plus("reactor.set_leader", False, opts_overrides=opts_overrides) ret = self.run_run_plus("reactor.is_leader", opts_overrides=opts_overrides) self.assertFalse(ret["return"]) try: master_event = self.get_event() self.fire_event({"id": "minion"}, "salt/test/reactor") while True: event = master_event.get_event(full=True) if event is None: continue if event.get("tag") == "test_reaction": # if we reach this point, the test is a failure self.assertTrue(True) # pylint: disable=redundant-unittest-assert break except TimeoutException as exc: self.assertTrue("Timeout" in str(exc)) finally: signal.alarm(0) # make reactor the leader again # ensure reactor engine is available opts_overrides = { "engines": [{ "reactor": { "refresh_interval": 60, "worker_threads": 10, "worker_hwm": 10000, } }] } self.run_run_plus("reactor.set_leader", True, opts_overrides=opts_overrides) ret = self.run_run_plus("reactor.is_leader", opts_overrides=opts_overrides) self.assertTrue(ret["return"]) # trigger a reaction signal.alarm(self.timeout) try: master_event = self.get_event() self.fire_event({"id": "minion"}, "salt/test/reactor") while True: event = master_event.get_event(full=True) if event is None: continue if event.get("tag") == "test_reaction": self.assertTrue(event["data"]["test_reaction"]) break finally: signal.alarm(0)