def _proc_runner(self, fun, low, user, tag, jid): ''' Run this method in a multiprocess target to execute the runner in a multiprocess and fire the return data on the event bus ''' salt.utils.daemonize() event = salt.utils.event.MasterEvent(self.opts['sock_dir']) data = {'fun': 'runner.{0}'.format(fun), 'jid': jid, 'user': user, } event.fire_event(data, tagify('new', base=tag)) try: data['return'] = self.low(fun, low) data['success'] = True except Exception as exc: data['return'] = 'Exception occured in runner {0}: {1}: {2}'.format( fun, exc.__class__.__name__, exc, ) data['success'] = False data['user'] = user event.fire_event(data, tagify('ret', base=tag)) # this is a workaround because process reaping is defeating 0MQ linger time.sleep(2.0) # delay so 0MQ event gets out before runner process reaped
def _proc_function(self, fun, low, user, tag, jid): ''' Run this method in a multiprocess target to execute the function in a multiprocess and fire the return data on the event bus ''' salt.utils.daemonize() data = {'fun': '{0}.{1}'.format(self.client, fun), 'jid': jid, 'user': user, } event = salt.utils.event.get_event( 'master', self.opts['sock_dir'], self.opts['transport'], opts=self.opts, listen=False) event.fire_event(data, tagify('new', base=tag)) try: data['return'] = self.low(fun, low) data['success'] = True except Exception as exc: data['return'] = 'Exception occurred in {0} {1}: {2}: {3}'.format( self.client, fun, exc.__class__.__name__, exc, ) data['success'] = False data['user'] = user event.fire_event(data, tagify('ret', base=tag)) # if we fired an event, make sure to delete the event object. # This will ensure that we call destroy, which will do the 0MQ linger del event
def _disbatch_local(self, chunk): ''' Disbatch local client commands ''' chunk_ret = {} f_call = salt.utils.format_call(self.saltclients['local'], chunk) # fire a job off try: ping_pub_data = self.saltclients['local'](chunk['tgt'], 'test.ping', [], expr_form=f_call['kwargs']['expr_form']) pub_data = self.saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {})) except EauthAuthenticationError: raise tornado.gen.Return('Not authorized to run this job') # if the job didn't publish, lets not wait around for nothing # TODO: set header?? if 'jid' not in pub_data: raise tornado.gen.Return('No minions matched the target. No command was sent, no jid was assigned.') # get the tag that we are looking for ping_tag = tagify([ping_pub_data['jid'], 'ret'], 'job') ret_tag = tagify([pub_data['jid'], 'ret'], 'job') # seed minions_remaining with the pub_data minions_remaining = pub_data['minions'] ret_event = self.application.event_listener.get_event(self, tag=ret_tag) ping_event = self.application.event_listener.get_event(self, tag=ping_tag) # while we are waiting on all the mininons while len(minions_remaining) > 0 or not self.min_syndic_wait_done(): event_future = yield Any([ret_event, ping_event]) try: event = event_future.result() # if you hit a timeout, just stop waiting ;) except TimeoutException: break # If someone returned from the ping, and they are new-- add to minions_remaining if event_future == ping_event: ping_id = event['data']['id'] if ping_id not in chunk_ret and ping_id not in minions_remaining: minions_remaining.append(ping_id) ping_event = self.application.event_listener.get_event(self, tag=ping_tag) # if it is a ret future, its just a regular return else: chunk_ret[event['data']['id']] = event['data']['return'] # its possible to get a return that wasn't in the minion_remaining list try: minions_remaining.remove(event['data']['id']) except ValueError: pass ret_event = self.application.event_listener.get_event(self, tag=ret_tag) raise tornado.gen.Return(chunk_ret)
def testMinionStatsWrongMissingTag(self): """ Test Minion Stats requests with unknown and missing tag (A3, A4) """ console.terse("{0}\n".format(self.testMinionStatsWrongMissingTag.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMinion") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("SaltRaetRoadStackSetup") self.addEnterDeed("StatsMinionTestSetup") act = self.addRecurDeed("SaltRaetStatsEventerMinion") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add a test stat key-value roadStack = self.store.fetch('.salt.road.manor.stack') laneStack = self.store.fetch('.salt.lane.manor.stack') roadStack.value.stats = odict({'test_road_stats_event': 111}) laneStack.value.stats = odict({'test_lane_stats_event': 222}) # ensure stats are equal to expected self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) # add stats request testStack = self.store.fetch('.salt.test.road.stack').value statsReq = self.store.fetch('.salt.stats.event_req').value tag = 'salt/unknown/tag' self.assertNotEqual(tag, tagify('lane', 'stats')) self.assertNotEqual(tag, tagify('road', 'stats')) minionName = roadStack.value.local.name masterName = testStack.local.name # unknown tag in stats request statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), 'src': (masterName, None, None)}, 'tag': tag}) # no tag in stats request statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), 'src': (masterName, None, None)}}) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 0) # Close active stacks servers act.actor.lane_stack.value.server.close() act.actor.road_stack.value.server.close() testStack = self.store.fetch('.salt.test.road.stack') if testStack: testStack.value.server.close()
def update(): ''' Execute an hg pull on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'hgfs'} pid = os.getpid() data['changed'] = purge_cache() for repo in init(): repo['repo'].open() lk_fn = os.path.join(repo['repo'].root(), 'update.lk') with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write(str(pid)) curtip = repo['repo'].tip() try: repo['repo'].pull() except Exception as exc: log.error( 'Exception {0} caught while updating hgfs remote {1}' .format(exc, repo['uri']), exc_info=log.isEnabledFor(logging.DEBUG) ) else: newtip = repo['repo'].tip() if curtip[1] != newtip[1]: data['changed'] = True repo['repo'].close() try: os.remove(lk_fn) except (IOError, OSError): pass env_cache = os.path.join(__opts__['cachedir'], 'hgfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): env_cachedir = os.path.dirname(env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.fopen(env_cache, 'w+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) # if there is a change, fire an event if __opts__.get('fileserver_events', False): event = salt.utils.event.get_event( 'master', __opts__['sock_dir'], __opts__['transport'], listen=False) event.fire_event(data, tagify(['hgfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'hgfs/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def all_returns(self, jid, finish_futures=None, minions_remaining=None, ): ''' Return a future which will complete once all returns are completed (according to minions_remaining), or one of the passed in "finish_futures" completes ''' if finish_futures is None: finish_futures = [] if minions_remaining is None: minions_remaining = [] ret_tag = tagify([jid, 'ret'], 'job') chunk_ret = {} while True: ret_event = self.application.event_listener.get_event(self, tag=ret_tag, ) f = yield Any([ret_event] + finish_futures) if f in finish_futures: raise tornado.gen.Return(chunk_ret) event = f.result() chunk_ret[event['data']['id']] = event['data']['return'] # its possible to get a return that wasn't in the minion_remaining list try: minions_remaining.remove(event['data']['id']) except ValueError: pass if len(minions_remaining) == 0: raise tornado.gen.Return(chunk_ret)
def delete_key(self, match=None, match_dict=None): ''' Delete public keys. If "match" is passed, it is evaluated as a glob. Pre-gathered matches can also be passed via "match_dict". ''' if match is not None: matches = self.name_match(match) elif match_dict is not None and isinstance(match_dict, dict): matches = match_dict else: matches = {} for status, keys in matches.items(): for key in keys: try: os.remove(os.path.join(self.opts['pki_dir'], status, key)) eload = {'result': True, 'act': 'delete', 'id': key} self.event.fire_event(eload, tagify(prefix='key')) except (OSError, IOError): pass self.check_minion_cache() salt.crypt.dropfile(self.opts['cachedir'], self.opts['user']) return ( self.name_match(match) if match is not None else self.dict_match(matches) )
def process_queue(queue, quantity=1, backend='sqlite'): ''' Pop items off a queue and create an event on the Salt event bus to be processed by a Reactor. CLI Example: .. code-block:: bash salt-run queue.process_queue myqueue salt-run queue.process_queue myqueue 6 salt-run queue.process_queue myqueue all backend=sqlite ''' # get ready to send an event event = salt.utils.event.get_event( 'master', __opts__['sock_dir'], __opts__['transport'], opts=__opts__, listen=False) try: items = pop(queue=queue, quantity=quantity, backend=backend) except SaltInvocationError as exc: error_txt = '{0}'.format(exc) __progress__(error_txt) return False data = {'items': items, 'backend': backend, 'queue': queue, } event.fire_event(data, tagify([queue, 'process'], prefix='queue'))
def get_stats(estate=None, stack="road"): """ Print the stack stats estate : None The name of the target estate. Master stats would be requested by default stack : 'road' Show stats on either road or lane stack Allowed values are 'road' or 'lane'. CLI Example: .. code-block:: bash salt-run manage.get_stats [estate=alpha_minion] [stack=lane] """ conf_file = __opts__["conf_file"] opts = salt.config.client_config(conf_file) if opts["transport"] == "raet": tag = tagify(stack, "stats") event = salt.utils.raetevent.StatsEvent(__opts__, __opts__["sock_dir"], tag=tag, estate=estate) stats = event.get_event(wait=60, tag=tag) else: # TODO: implement 0MQ analog stats = "Not implemented" return stats
def update(self): """ COPIED FROM SALT changed: salt.utils.fopen() call opens the file in binary mode instead. """ # data for the fileserver event data = {"changed": self.clear_old_remotes(), "backend": "gitfs"} if self.fetch_remotes(): data["changed"] = True if data["changed"] is True or not os.path.isfile(self.env_cache): env_cachedir = os.path.dirname(self.env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = self.envs(ignore_cache=True) serial = salt.payload.Serial(self.opts) with salt.utils.fopen(self.env_cache, "wb+") as fp_: fp_.write(serial.dumps(new_envs)) logger.trace("Wrote env cache data to {0}".format(self.env_cache)) # if there is a change, fire an event if self.opts.get("fileserver_events", False): event = salt.utils.event.get_event( "master", self.opts["sock_dir"], self.opts["transport"], opts=self.opts, listen=False ) event.fire_event(data, tagify(["gitfs", "update"], prefix="fileserver")) try: salt.fileserver.reap_fileserver_cache_dir(self.hash_cachedir, self.find_file) except (OSError, IOError): # Hash file won't exist if no files have yet been served up pass
def cmd_sync(self, low, timeout=None): ''' Execute a runner function synchronously; eauth is respected This function requires that :conf_master:`external_auth` is configured and the user is authorized to execute runner functions: (``@runner``). .. code-block:: python runner.eauth_sync({ 'fun': 'jobs.list_jobs', 'username': '******', 'password': '******', 'eauth': 'pam', }) ''' reformatted_low = self._reformat_low(low) job = self.master_call(**reformatted_low) ret_tag = tagify('ret', base=job['tag']) timelimit = time.time() + (timeout or 300) while True: ret = self.event.get_event(full=True) if ret is None: if time.time() > timelimit: raise salt.exceptions.SaltClientTimeout( "RunnerClient job '{0}' timed out".format(job['jid']), jid=job['jid']) else: continue if ret['tag'] == ret_tag: return ret['data']['return']
def update(): ''' Execute an svn update on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'svnfs'} pid = os.getpid() data['changed'] = purge_cache() for repo in init(): lk_fn = os.path.join(repo['repo'], 'update.lk') with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write(str(pid)) old_rev = _rev(repo) try: CLIENT.update(repo['repo']) except pysvn._pysvn.ClientError as exc: log.error( 'Error updating svnfs remote {0} (cachedir: {1}): {2}' .format(repo['uri'], repo['cachedir'], exc) ) try: os.remove(lk_fn) except (OSError, IOError): pass new_rev = _rev(repo) if any((x is None for x in (old_rev, new_rev))): # There were problems getting the revision ID continue if new_rev != old_rev: data['changed'] = True env_cache = os.path.join(__opts__['cachedir'], 'svnfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): env_cachedir = os.path.dirname(env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.fopen(env_cache, 'w+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) # if there is a change, fire an event if __opts__.get('fileserver_events', False): event = salt.utils.event.get_event( 'master', __opts__['sock_dir'], __opts__['transport'], listen=False) event.fire_event(data, tagify(['svnfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'svnfs/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def accept(self, match): ''' Accept a specified host's public key based on name or keys based on glob ''' matches = self.name_match(match) if 'minions_pre' in matches: for key in matches['minions_pre']: try: shutil.move( os.path.join( self.opts['pki_dir'], 'minions_pre', key), os.path.join( self.opts['pki_dir'], 'minions', key) ) eload = {'result': True, 'act': 'accept', 'id': key} self.event.fire_event(eload, tagify(prefix='key')) except (IOError, OSError): pass return self.name_match(match)
def reject(self, match): ''' Reject a specified host's public key or keys based on a glob ''' matches = self.name_match(match) if 'minions_pre' in matches: for key in matches['minions_pre']: try: shutil.move( os.path.join( self.opts['pki_dir'], 'minions_pre', key), os.path.join( self.opts['pki_dir'], 'minions_rejected', key) ) eload = {'result': True, 'act': 'reject', 'id': key} self.event.fire_event(eload, tagify(prefix='key')) except (IOError, OSError): pass self.check_minion_cache() salt.crypt.dropfile(self.opts['cachedir'], self.opts['user']) return self.name_match(match)
def reject_all(self): ''' Reject all keys in pre ''' keys = self.list_keys() for key in keys['minions_pre']: try: shutil.move( os.path.join( self.opts['pki_dir'], 'minions_pre', key), os.path.join( self.opts['pki_dir'], 'minions_rejected', key) ) eload = {'result': True, 'act': 'reject', 'id': key} self.event.fire_event(eload, tagify(prefix='key')) except (IOError, OSError): pass self.check_minion_cache() salt.crypt.dropfile(self.opts['cachedir'], self.opts['user']) return self.list_keys()
def get_stats(estate=None, stack='road'): ''' Print the stack stats estate : None The name of the target estate. Master stats would be requested by default stack : 'road' Show stats on either road or lane stack Allowed values are 'road' or 'lane'. CLI Example: .. code-block:: bash salt-run manage.get_stats [estate=alpha_minion] [stack=lane] ''' conf_file = __opts__['conf_file'] opts = salt.config.client_config(conf_file) if opts['transport'] == 'raet': tag = tagify(stack, 'stats') event = salt.utils.raetevent.StatsEvent(__opts__, __opts__['sock_dir'], tag=tag, estate=estate) stats = event.get_event(wait=60, tag=tag) else: #TODO: implement 0MQ analog stats = 'Not implemented' return stats
def fire_event(self, data, tag): ''' fires event with data and tag This only works if api is running with same user permissions as master Need to convert this to a master call with appropriate authentication ''' return self.event.fire_event(data, tagify(tag, 'wui'))
def update(): ''' Execute a git pull on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'gitfs'} provider = _get_provider() pid = os.getpid() data['changed'] = purge_cache() repos = init() for repo in repos: origin = repo.remotes[0] if provider == 'gitpython': working_dir = repo.working_dir elif provider == 'pygit2': working_dir = repo.workdir lk_fn = os.path.join(working_dir, 'update.lk') with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write(str(pid)) try: if provider == 'gitpython': for fetch in origin.fetch(): if fetch.old_commit is not None: data['changed'] = True elif provider == 'pygit2': fetch = origin.fetch() if fetch.get('received_objects', 0): data['changed'] = True except Exception as exc: log.warning( 'Exception caught while fetching: {0}'.format(exc) ) try: os.remove(lk_fn) except (IOError, OSError): pass env_cache = os.path.join(__opts__['cachedir'], 'gitfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.fopen(env_cache, 'w+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) # if there is a change, fire an event if __opts__.get('fileserver_events', False): event = salt.utils.event.MasterEvent(__opts__['sock_dir']) event.fire_event(data, tagify(['gitfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'gitfs/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def job_not_running(self, jid, tgt, tgt_type, minions_remaining=None, ): ''' Return a future which will complete once jid (passed in) is no longer running on tgt ''' if minions_remaining is None: minions_remaining = [] ping_pub_data = self.saltclients['local'](tgt, 'saltutil.find_job', [jid], expr_form=tgt_type) ping_tag = tagify([ping_pub_data['jid'], 'ret'], 'job') minion_running = False while True: try: event = yield self.application.event_listener.get_event(self, tag=ping_tag, timeout=self.application.opts['gather_job_timeout'], ) except TimeoutException: if not minion_running: raise tornado.gen.Return(True) else: ping_pub_data = self.saltclients['local'](tgt, 'saltutil.find_job', [jid], expr_form=tgt_type) ping_tag = tagify([ping_pub_data['jid'], 'ret'], 'job') minion_running = False continue # Minions can return, we want to see if the job is running... if event['data'].get('return', {}) == {}: continue minion_running = True id_ = event['data']['id'] if id_ not in minions_remaining: minions_remaining.append(event['data']['id'])
def testMinionLaneStats(self): """ Test Minion Road Stats request (A2) """ console.terse("{0}\n".format(self.testMinionLaneStats.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMinion") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("SaltRaetRoadStackSetup") self.addEnterDeed("StatsMinionTestSetup") act = self.addRecurDeed("SaltRaetStatsEventerMinion") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add a test stat key-value roadStack = self.store.fetch('.salt.road.manor.stack') laneStack = self.store.fetch('.salt.lane.manor.stack') roadStack.value.stats = odict() laneStack.value.stats = odict({'test_stats_event': 111}) # ensure stats are equal to expected self.assertDictEqual(roadStack.value.stats, {}) self.assertDictEqual(laneStack.value.stats, {'test_stats_event': 111}) # add stats request testStack = self.store.fetch('.salt.test.road.stack').value statsReq = self.store.fetch('.salt.stats.event_req').value tag = tagify('lane', 'stats') minionName = roadStack.value.local.name masterName = testStack.local.name # lane stats request statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), 'src': (masterName, None, None)}, 'tag': tag}) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 1) msg, sender = testStack.rxMsgs.popleft() self.assertDictEqual(msg, {u'route': {u'src': [ns2u(minionName), u'manor', None], u'dst': [ns2u(masterName), None, u'event_fire']}, u'tag': ns2u(tag), u'data': {u'test_stats_event': 111}}) # Close active stacks servers act.actor.lane_stack.value.server.close() act.actor.road_stack.value.server.close() testStack = self.store.fetch('.salt.test.road.stack') if testStack: testStack.value.server.close()
def update(): ''' When we are asked to update (regular interval) lets reap the cache ''' try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'roots/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass mtime_map_path = os.path.join(__opts__['cachedir'], 'roots/mtime_map') # data to send on event data = {'changed': False, 'backend': 'roots'} old_mtime_map = {} # if you have an old map, load that if os.path.exists(mtime_map_path): with salt.utils.fopen(mtime_map_path, 'r') as fp_: for line in fp_: try: file_path, mtime = line.split(':', 1) old_mtime_map[file_path] = mtime except ValueError: # Document the invalid entry in the log log.warning('Skipped invalid cache mtime entry in {0}: {1}' .format(mtime_map_path, line)) # generate the new map new_mtime_map = salt.fileserver.generate_mtime_map(__opts__['file_roots']) # compare the maps, set changed to the return value data['changed'] = salt.fileserver.diff_mtime_map(old_mtime_map, new_mtime_map) # write out the new map mtime_map_path_dir = os.path.dirname(mtime_map_path) if not os.path.exists(mtime_map_path_dir): os.makedirs(mtime_map_path_dir) with salt.utils.fopen(mtime_map_path, 'w') as fp_: for file_path, mtime in six.iteritems(new_mtime_map): fp_.write('{file_path}:{mtime}\n'.format(file_path=file_path, mtime=mtime)) if __opts__.get('fileserver_events', False): # if there is a change, fire an event event = salt.utils.event.get_event( 'master', __opts__['sock_dir'], __opts__['transport'], opts=__opts__, listen=False) event.fire_event(data, tagify(['roots', 'update'], prefix='fileserver'))
def _disbatch_local_batch(self, chunk): ''' Disbatch local client batched commands ''' f_call = salt.utils.format_call(self.saltclients['local_batch'], chunk) # ping all the minions (to see who we have to talk to) # Don't catch any exception, since we won't know what to do, we'll # let the upper level deal with this one ping_ret = yield self._disbatch_local({'tgt': chunk['tgt'], 'fun': 'test.ping', 'expr_form': f_call['kwargs']['expr_form']}) chunk_ret = {} if not isinstance(ping_ret, dict): raise tornado.gen.Return(chunk_ret) minions = ping_ret.keys() maxflight = get_batch_size(f_call['kwargs']['batch'], len(minions)) inflight_futures = [] # override the expr_form f_call['kwargs']['expr_form'] = 'list' # do this batch while len(minions) > 0 or len(inflight_futures) > 0: # if you have more to go, lets disbatch jobs while len(inflight_futures) < maxflight and len(minions) > 0: minion_id = minions.pop(0) f_call['args'][0] = [minion_id] # set the tgt to the minion pub_data = self.saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {})) # if the job didn't publish, lets not wait around for nothing # we'll just skip # TODO: set header??, some special return?, Or just ignore it (like we do in CLI) if 'jid' not in pub_data: continue tag = tagify([pub_data['jid'], 'ret', minion_id], 'job') future = self.application.event_listener.get_event(self, tag=tag) inflight_futures.append(future) # if we have nothing to wait for, don't wait if len(inflight_futures) == 0: continue # wait until someone is done finished_future = yield Any(inflight_futures) try: event = finished_future.result() except TimeoutException: break chunk_ret[event['data']['id']] = event['data']['return'] inflight_futures.remove(finished_future) raise tornado.gen.Return(chunk_ret)
def update(): ''' When we are asked to update (regular interval) lets reap the cache ''' try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'stackdio/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass mtime_map_path = os.path.join(__opts__['cachedir'], 'stackdio/mtime_map') # data to send on event data = {'changed': False, 'backend': 'stackdio'} old_mtime_map = {} # if you have an old map, load that if os.path.exists(mtime_map_path): with salt.utils.fopen(mtime_map_path, 'rb') as fp_: for line in fp_: file_path, mtime = line.split(':', 1) old_mtime_map[file_path] = mtime # generate the new map user_envs_dir = get_envs_dir() user_envs = envs() path_map = {} for user_env in user_envs: path_map[user_env] = [os.path.join(user_envs_dir, user_env)] new_mtime_map = salt.fileserver.generate_mtime_map(path_map) # compare the maps, set changed to the return value data['changed'] = salt.fileserver.diff_mtime_map(old_mtime_map, new_mtime_map) # write out the new map mtime_map_path_dir = os.path.dirname(mtime_map_path) if not os.path.exists(mtime_map_path_dir): os.makedirs(mtime_map_path_dir) with salt.utils.fopen(mtime_map_path, 'w') as fp_: for file_path, mtime in new_mtime_map.iteritems(): fp_.write('{file_path}:{mtime}\n'.format(file_path=file_path, mtime=mtime)) if __opts__.get('fileserver_events', False): # if there is a change, fire an event event = salt.utils.event.MasterEvent(__opts__['sock_dir']) event.fire_event( data, tagify(['stackdio', 'update'], prefix='fileserver') )
def action(self): testStack = self.event_stack.value self.assertTrue(len(testStack.rxMsgs) == 0) testStack.serviceAll() self.assertTrue(len(testStack.rxMsgs) == 1) tag = tagify('present', 'presence') msg, sender = testStack.rxMsgs.popleft() self.assertTrue(msg == {'route': {'src': [None, 'manor', None], 'dst': [None, None, 'event_fire']}, 'tag': tag, 'data': {'allowed': {'alpha':'1.1.1.1'}}})
def update(): ''' Execute a hg pull on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'hgfs'} pid = os.getpid() data['changed'] = purge_cache() repos = init() for repo in repos: repo.open() lk_fn = os.path.join(repo.root(), 'update.lk') with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write(str(pid)) curtip = repo.tip() try: success = repo.pull() except Exception as exc: log.error( 'Exception caught while updating hgfs: {0}'.format(exc) ) else: newtip = repo.tip() if curtip[1] != newtip[1]: data['changed'] = True repo.close() try: os.remove(lk_fn) except (IOError, OSError): pass env_cache = os.path.join(__opts__['cachedir'], 'hgfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.fopen(env_cache, 'w+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) # if there is a change, fire an event if __opts__.get('fileserver_events', False): event = salt.utils.event.MasterEvent(__opts__['sock_dir']) event.fire_event(data, tagify(['hgfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'hgfs/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def _disbatch_local_batch(self): ''' Disbatch local client batched commands ''' self.ret = [] for chunk in self.lowstate: f_call = salt.utils.format_call(self.saltclients['local_batch'], chunk) timeout = float(chunk.get('timeout', self.application.opts['timeout'])) # set the timeout timeout_obj = tornado.ioloop.IOLoop.instance().add_timeout(time.time() + timeout, self.timeout_futures) # ping all the minions (to see who we have to talk to) # TODO: actually ping them all? this just gets the pub data minions = self.saltclients['local'](chunk['tgt'], 'test.ping', [], expr_form=f_call['kwargs']['expr_form'])['minions'] chunk_ret = {} maxflight = get_batch_size(f_call['kwargs']['batch'], len(minions)) inflight_futures = [] # do this batch while len(minions) > 0: # if you have more to go, lets disbatch jobs while len(inflight_futures) < maxflight: minion_id = minions.pop(0) f_call['args'][0] = minion_id # TODO: list?? f_call['kwargs']['expr_form'] = 'glob' pub_data = self.saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {})) tag = tagify([pub_data['jid'], 'ret', minion_id], 'job') future = self.application.event_listener.get_event(self, tag=tag) inflight_futures.append(future) # wait until someone is done finished_future = yield Any(inflight_futures) try: event = finished_future.result() except TimeoutException: break chunk_ret[event['data']['id']] = event['data']['return'] inflight_futures.remove(finished_future) self.ret.append(chunk_ret) # if we finish in time, cancel the timeout tornado.ioloop.IOLoop.instance().remove_timeout(timeout_obj) self.write(self.serialize({'return': self.ret})) self.finish()
def _proc_runner(self, tag, fun, low): ''' Run this method in a multiprocess target to execute the runner in a multiprocess and fire the return data on the event bus ''' salt.utils.daemonize() event = salt.utils.event.MasterEvent(self.opts['sock_dir']) data = {'fun': "runner.{0}".format(fun), 'jid': low['jid'], } event.fire_event(data, tagify('new', base=tag)) try: data['ret'] = self.low(fun, low) data['success'] = True except Exception as exc: data['ret'] = 'Exception occured in runner {0}: {1}'.format( fun, exc, ) event.fire_event(data, tagify('ret', base=tag))
def testPresenceAvailableSomeIpUnknown(self): """ Test Presenter 'available' request with some minion addresses aren't known (D3) """ console.terse("{0}\n".format(self.testPresenceAvailableSomeIpUnknown.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMaster") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("PresenterTestSetup") act = self.addRecurDeed("SaltRaetPresenter") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add available minions self.addAvailable('alpha') self.addAvailable('beta') self.addAvailable('gamma') self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234') self.addPresenceInfo('aliveds', 'delta', '1.2.3.4', '1234') # add presence request testStack = self.store.fetch('.salt.test.lane.stack').value presenceReq = self.store.fetch('.salt.presence.event_req').value ryn = 'manor' presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None)}, 'data': {'state': 'available'}}) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 1) tag = tagify('present', 'presence') msg, sender = testStack.rxMsgs.popleft() self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], 'dst': [None, None, 'event_fire']}, 'tag': tag, 'data': {'present': {'alpha': '1.1.1.1', 'beta': None, 'gamma': None}}}) # Close active stacks servers act.actor.lane_stack.value.server.close() testStack = self.store.fetch('.salt.test.lane.stack') if testStack: testStack.value.server.close()
def testPresenceJoined(self): """ Test Presenter 'joined' request (A2) """ console.terse("{0}\n".format(self.testPresenceJoined.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMaster") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("PresenterTestSetup") act = self.addRecurDeed("SaltRaetPresenter") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add joined minions # NOTE: for now alloweds are threaded as joineds self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234') self.addPresenceInfo('alloweds', 'beta', '1.2.3.4', '1234') # add presence request testStack = self.store.fetch('.salt.test.lane.stack').value presenceReq = self.store.fetch('.salt.presence.event_req').value ryn = 'manor' msg = {'route': {'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None)}, 'data': {'state': 'joined'}} presenceReq.append(msg) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 1) tag = tagify('present', 'presence') msg, sender = testStack.rxMsgs.popleft() self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], 'dst': [None, None, 'event_fire']}, 'tag': tag, 'data': {'joined': {'alpha': '1.1.1.1', 'beta': '1.2.3.4'}}}) # Close active stacks servers act.actor.lane_stack.value.server.close() testStack = self.store.fetch('.salt.test.lane.stack') if testStack: testStack.value.server.close()
def testMasterStatsUnknownRemote(self): """ Test Master Stats request with unknown remote (B1) """ console.terse("{0}\n".format(self.testMasterStatsUnknownRemote.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMaster") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("SaltRaetRoadStackSetup") self.addEnterDeed("StatsMasterTestSetup") act = self.addRecurDeed("SaltRaetStatsEventerMaster") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add a test stat key-value roadStack = self.store.fetch('.salt.road.manor.stack') laneStack = self.store.fetch('.salt.lane.manor.stack') roadStack.value.stats['test_road_stats_event'] = 111 laneStack.value.stats['test_lane_stats_event'] = 222 # ensure stats are equal to expected self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) # add stats request testStack = self.store.fetch('.salt.test.lane.stack').value statsReq = self.store.fetch('.salt.stats.event_req').value tag = tagify('road', 'stats') # unknown tag in stats request unknownName = 'unknownName' statsReq.append({'route': {'dst': (None, None, 'stats_req'), 'src': (None, unknownName, None)}, 'tag': tag}) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 0) # Close active stacks servers act.actor.lane_stack.value.server.close() act.actor.road_stack.value.server.close() testStack = self.store.fetch('.salt.test.lane.stack') if testStack: testStack.value.server.close()
def thread_return(cls, minion_instance, opts, data): ''' This method should be used as a threading target, start the actual minion side execution. ''' fn_ = os.path.join(minion_instance.proc_dir, data['jid']) salt.utils.process.appendproctitle('{0}._thread_return {1}'.format( cls.__name__, data['jid'])) sdata = {'pid': os.getpid()} sdata.update(data) log.info('Starting a new job with PID %s', sdata['pid']) with salt.utils.files.fopen(fn_, 'w+b') as fp_: fp_.write(minion_instance.serial.dumps(sdata)) ret = {'success': False} function_name = data['fun'] executors = data.get('module_executors') or \ getattr(minion_instance, 'module_executors', []) or \ opts.get('module_executors', ['direct_call']) allow_missing_funcs = any([ minion_instance.executors['{0}.allow_missing_func'.format(executor)]( function_name) for executor in executors if '{0}.allow_missing_func' in minion_instance.executors ]) if function_name in minion_instance.functions or allow_missing_funcs is True: try: minion_blackout_violation = False if minion_instance.connected and minion_instance.opts[ 'pillar'].get('minion_blackout', False): whitelist = minion_instance.opts['pillar'].get( 'minion_blackout_whitelist', []) # this minion is blacked out. Only allow saltutil.refresh_pillar and the whitelist if function_name != 'saltutil.refresh_pillar' and function_name not in whitelist: minion_blackout_violation = True # use minion_blackout_whitelist from grains if it exists if minion_instance.opts['grains'].get('minion_blackout', False): whitelist = minion_instance.opts['grains'].get( 'minion_blackout_whitelist', []) if function_name != 'saltutil.refresh_pillar' and function_name not in whitelist: minion_blackout_violation = True if minion_blackout_violation: raise SaltInvocationError( 'Minion in blackout mode. Set \'minion_blackout\' ' 'to False in pillar or grains to resume operations. Only ' 'saltutil.refresh_pillar allowed in blackout mode.') if function_name in minion_instance.functions: func = minion_instance.functions[function_name] args, kwargs = salt.minion.load_args_and_kwargs( func, data['arg'], data) else: # only run if function_name is not in minion_instance.functions and allow_missing_funcs is True func = function_name args, kwargs = data['arg'], data minion_instance.functions.pack['__context__']['retcode'] = 0 if isinstance(executors, six.string_types): executors = [executors] elif not isinstance(executors, list) or not executors: raise SaltInvocationError( "Wrong executors specification: {0}. String or non-empty list expected" .format(executors)) if opts.get('sudo_user', '') and executors[-1] != 'sudo': executors[-1] = 'sudo' # replace the last one with sudo log.trace('Executors list %s', executors) # pylint: disable=no-member for name in executors: fname = '{0}.execute'.format(name) if fname not in minion_instance.executors: raise SaltInvocationError( "Executor '{0}' is not available".format(name)) return_data = minion_instance.executors[fname](opts, data, func, args, kwargs) if return_data is not None: break if isinstance(return_data, types.GeneratorType): ind = 0 iret = {} for single in return_data: if isinstance(single, dict) and isinstance(iret, dict): iret.update(single) else: if not iret: iret = [] iret.append(single) tag = tagify( [data['jid'], 'prog', opts['id'], six.text_type(ind)], 'job') event_data = {'return': single} minion_instance._fire_master(event_data, tag) ind += 1 ret['return'] = iret else: ret['return'] = return_data retcode = minion_instance.functions.pack['__context__'].get( 'retcode', salt.defaults.exitcodes.EX_OK) if retcode == salt.defaults.exitcodes.EX_OK: # No nonzero retcode in __context__ dunder. Check if return # is a dictionary with a "result" or "success" key. try: func_result = all( return_data.get(x, True) for x in ('result', 'success')) except Exception: # return data is not a dict func_result = True if not func_result: retcode = salt.defaults.exitcodes.EX_GENERIC ret['retcode'] = retcode ret['success'] = retcode == salt.defaults.exitcodes.EX_OK except CommandNotFoundError as exc: msg = 'Command required for \'{0}\' not found'.format( function_name) log.debug(msg, exc_info=True) ret['return'] = '{0}: {1}'.format(msg, exc) ret['out'] = 'nested' ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC except CommandExecutionError as exc: log.error('A command in \'%s\' had a problem: %s', function_name, exc, exc_info_on_loglevel=logging.DEBUG) ret['return'] = 'ERROR: {0}'.format(exc) ret['out'] = 'nested' ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC except SaltInvocationError as exc: log.error('Problem executing \'%s\': %s', function_name, exc, exc_info_on_loglevel=logging.DEBUG) ret['return'] = 'ERROR executing \'{0}\': {1}'.format( function_name, exc) ret['out'] = 'nested' ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC except TypeError as exc: msg = 'Passed invalid arguments to {0}: {1}\n{2}'.format( function_name, exc, func.__doc__ or '') log.warning(msg, exc_info_on_loglevel=logging.DEBUG) ret['return'] = msg ret['out'] = 'nested' ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC except Exception: msg = 'The minion function caused an exception' log.warning(msg, exc_info_on_loglevel=True) salt.utils.error.fire_exception(salt.exceptions.MinionError(msg), opts, job=data) ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc()) ret['out'] = 'nested' ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC else: docs = minion_instance.functions['sys.doc']( '{0}*'.format(function_name)) if docs: docs[function_name] = minion_instance.functions.missing_fun_string( function_name) ret['return'] = docs else: ret['return'] = minion_instance.functions.missing_fun_string( function_name) mod_name = function_name.split('.')[0] if mod_name in minion_instance.function_errors: ret['return'] += ' Possible reasons: \'{0}\''.format( minion_instance.function_errors[mod_name]) ret['success'] = False ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC ret['out'] = 'nested' ret['jid'] = data['jid'] ret['fun'] = data['fun'] ret['fun_args'] = data['arg'] if 'master_id' in data: ret['master_id'] = data['master_id'] if 'metadata' in data: if isinstance(data['metadata'], dict): ret['metadata'] = data['metadata'] else: log.warning( 'The metadata parameter must be a dictionary. Ignoring.') if minion_instance.connected: minion_instance._return_pub( ret, timeout=minion_instance._return_retry_timer()) # Add default returners from minion config # Should have been coverted to comma-delimited string already if isinstance(opts.get('return'), six.string_types): if data['ret']: data['ret'] = ','.join((data['ret'], opts['return'])) else: data['ret'] = opts['return'] log.debug('minion return: %s', ret) # TODO: make a list? Seems odd to split it this late :/ if data['ret'] and isinstance(data['ret'], six.string_types): if 'ret_config' in data: ret['ret_config'] = data['ret_config'] if 'ret_kwargs' in data: ret['ret_kwargs'] = data['ret_kwargs'] ret['id'] = opts['id'] for returner in set(data['ret'].split(',')): try: returner_str = '{0}.returner'.format(returner) if returner_str in minion_instance.returners: minion_instance.returners[returner_str](ret) else: returner_err = minion_instance.returners.missing_fun_string( returner_str) log.error('Returner %s could not be loaded: %s', returner_str, returner_err) except Exception as exc: log.exception('The return failed for job %s: %s', data['jid'], exc)
def _disbatch_local_batch(self, chunk): ''' Disbatch local client batched commands ''' f_call = salt.utils.format_call(self.saltclients['local_batch'], chunk) # ping all the minions (to see who we have to talk to) # Don't catch any exception, since we won't know what to do, we'll # let the upper level deal with this one ping_ret = yield self._disbatch_local({ 'tgt': chunk['tgt'], 'fun': 'test.ping', 'expr_form': f_call['kwargs']['expr_form'] }) chunk_ret = {} if not isinstance(ping_ret, dict): raise tornado.gen.Return(chunk_ret) minions = ping_ret.keys() maxflight = get_batch_size(f_call['kwargs']['batch'], len(minions)) inflight_futures = [] # override the expr_form f_call['kwargs']['expr_form'] = 'list' # do this batch while len(minions) > 0 or len(inflight_futures) > 0: # if you have more to go, lets disbatch jobs while len(inflight_futures) < maxflight and len(minions) > 0: minion_id = minions.pop(0) f_call['args'][0] = [minion_id] # set the tgt to the minion pub_data = self.saltclients['local'](*f_call.get('args', ()), **f_call.get( 'kwargs', {})) # if the job didn't publish, lets not wait around for nothing # we'll just skip # TODO: set header??, some special return?, Or just ignore it (like we do in CLI) if 'jid' not in pub_data: continue tag = tagify([pub_data['jid'], 'ret', minion_id], 'job') future = self.application.event_listener.get_event(self, tag=tag) inflight_futures.append(future) # if we have nothing to wait for, don't wait if len(inflight_futures) == 0: continue # wait until someone is done finished_future = yield Any(inflight_futures) try: event = finished_future.result() except TimeoutException: break chunk_ret[event['data']['id']] = event['data']['return'] inflight_futures.remove(finished_future) raise tornado.gen.Return(chunk_ret)
def _disbatch_local(self, chunk): ''' Disbatch local client commands ''' chunk_ret = {} f_call = salt.utils.format_call(self.saltclients['local'], chunk) # fire a job off try: ping_pub_data = self.saltclients['local']( chunk['tgt'], 'test.ping', [], expr_form=f_call['kwargs']['expr_form']) pub_data = self.saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {})) except EauthAuthenticationError: raise tornado.gen.Return('Not authorized to run this job') # if the job didn't publish, lets not wait around for nothing # TODO: set header?? if 'jid' not in pub_data: raise tornado.gen.Return( 'No minions matched the target. No command was sent, no jid was assigned.' ) # get the tag that we are looking for ping_tag = tagify([ping_pub_data['jid'], 'ret'], 'job') ret_tag = tagify([pub_data['jid'], 'ret'], 'job') # seed minions_remaining with the pub_data minions_remaining = pub_data['minions'] ret_event = self.application.event_listener.get_event(self, tag=ret_tag) ping_event = self.application.event_listener.get_event(self, tag=ping_tag) # while we are waiting on all the mininons while len(minions_remaining) > 0 or not self.min_syndic_wait_done(): event_future = yield Any([ret_event, ping_event]) try: event = event_future.result() # if you hit a timeout, just stop waiting ;) except TimeoutException: break # If someone returned from the ping, and they are new-- add to minions_remaining if event_future == ping_event: ping_id = event['data']['id'] if ping_id not in chunk_ret and ping_id not in minions_remaining: minions_remaining.append(ping_id) ping_event = self.application.event_listener.get_event( self, tag=ping_tag) # if it is a ret future, its just a regular return else: chunk_ret[event['data']['id']] = event['data']['return'] # its possible to get a return that wasn't in the minion_remaining list try: minions_remaining.remove(event['data']['id']) except ValueError: pass ret_event = self.application.event_listener.get_event( self, tag=ret_tag) raise tornado.gen.Return(chunk_ret)
def update(): ''' Execute an svn update on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'svnfs'} pid = os.getpid() data['changed'] = purge_cache() for repo in init(): lk_fn = os.path.join(repo['repo'], 'update.lk') with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write(str(pid)) old_rev = _rev(repo) try: CLIENT.update(repo['repo']) except pysvn._pysvn.ClientError as exc: log.error( 'Error updating svnfs remote {0} (cachedir: {1}): {2}' .format(repo['url'], repo['cachedir'], exc) ) try: os.remove(lk_fn) except (OSError, IOError): pass new_rev = _rev(repo) if any((x is None for x in (old_rev, new_rev))): # There were problems getting the revision ID continue if new_rev != old_rev: data['changed'] = True env_cache = os.path.join(__opts__['cachedir'], 'svnfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): env_cachedir = os.path.dirname(env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.fopen(env_cache, 'w+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) # if there is a change, fire an event if __opts__.get('fileserver_events', False): event = salt.utils.event.get_event( 'master', __opts__['sock_dir'], __opts__['transport'], opts=__opts__, listen=False) event.fire_event(data, tagify(['svnfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'svnfs/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def update(): """ Execute an svn update on all of the repos """ # data for the fileserver event data = {"changed": False, "backend": "svnfs"} # _clear_old_remotes runs init(), so use the value from there to avoid a # second init() data["changed"], repos = _clear_old_remotes() for repo in repos: if os.path.exists(repo["lockfile"]): log.warning( "Update lockfile is present for svnfs remote %s, skipping. " "If this warning persists, it is possible that the update " "process was interrupted. Removing %s or running " "'salt-run fileserver.clear_lock svnfs' will allow updates " "to continue for this remote.", repo["url"], repo["lockfile"], ) continue _, errors = lock(repo) if errors: log.error( "Unable to set update lock for svnfs remote %s, skipping.", repo["url"] ) continue log.debug("svnfs is fetching from %s", repo["url"]) old_rev = _rev(repo) try: CLIENT.update(repo["repo"]) except pysvn._pysvn.ClientError as exc: log.error( "Error updating svnfs remote %s (cachedir: %s): %s", repo["url"], repo["cachedir"], exc, ) new_rev = _rev(repo) if any((x is None for x in (old_rev, new_rev))): # There were problems getting the revision ID continue if new_rev != old_rev: data["changed"] = True clear_lock(repo) env_cache = os.path.join(__opts__["cachedir"], "svnfs/envs.p") if data.get("changed", False) is True or not os.path.isfile(env_cache): env_cachedir = os.path.dirname(env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.files.fopen(env_cache, "wb+") as fp_: fp_.write(serial.dumps(new_envs)) log.trace("Wrote env cache data to %s", env_cache) # if there is a change, fire an event if __opts__.get("fileserver_events", False): with salt.utils.event.get_event( "master", __opts__["sock_dir"], __opts__["transport"], opts=__opts__, listen=False, ) as event: event.fire_event(data, tagify(["svnfs", "update"], prefix="fileserver")) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__["cachedir"], "svnfs/hash"), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def testMinionStatsWrongMissingTag(self): ''' Test Minion Stats requests with unknown and missing tag (A3, A4) ''' console.terse("{0}\n".format( self.testMinionStatsWrongMissingTag.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMinion") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("SaltRaetRoadStackSetup") self.addEnterDeed("StatsMinionTestSetup") act = self.addRecurDeed("SaltRaetStatsEventerMinion") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add a test stat key-value roadStack = self.store.fetch('.salt.road.manor.stack') laneStack = self.store.fetch('.salt.lane.manor.stack') roadStack.value.stats = odict({'test_road_stats_event': 111}) laneStack.value.stats = odict({'test_lane_stats_event': 222}) # ensure stats are equal to expected self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) # add stats request testStack = self.store.fetch('.salt.test.road.stack').value statsReq = self.store.fetch('.salt.stats.event_req').value tag = 'salt/unknown/tag' self.assertNotEqual(tag, tagify('lane', 'stats')) self.assertNotEqual(tag, tagify('road', 'stats')) minionName = roadStack.value.local.name masterName = testStack.local.name # unknown tag in stats request statsReq.append({ 'route': { 'dst': (minionName, None, 'stats_req'), 'src': (masterName, None, None) }, 'tag': tag }) # no tag in stats request statsReq.append({ 'route': { 'dst': (minionName, None, 'stats_req'), 'src': (masterName, None, None) } }) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 0) # Close active stacks servers act.actor.lane_stack.value.server.close() act.actor.road_stack.value.server.close() testStack = self.store.fetch('.salt.test.road.stack') if testStack: testStack.value.server.close()
def _post_stats(self, stats): ''' Fire events with stat info if it's time ''' end_time = time.time() if end_time - self.stat_clock > self.opts['master_stats_event_iter']: # Fire the event with the stats and wipe the tracker self.event.fire_event({'time': end_time - self.stat_clock, 'worker': self.name, 'stats': stats}, tagify(self.name, 'stats')) self.stats = collections.defaultdict(lambda: {'mean': 0, 'latency': 0, 'runs': 0}) self.stat_clock = end_time
def proc_run(self, msg): ''' Execute the run in a dedicated process ''' data = msg['pub'] fn_ = os.path.join(self.proc_dir, data['jid']) self.opts['__ex_id'] = data['jid'] salt.utils.process.daemonize_if(self.opts) salt.transport.jobber_stack = stack = self._setup_jobber_stack() # set up return destination from source src_estate, src_yard, src_share = msg['route']['src'] salt.transport.jobber_estate_name = src_estate salt.transport.jobber_yard_name = src_yard sdata = {'pid': os.getpid()} sdata.update(data) with salt.utils.files.fopen(fn_, 'w+b') as fp_: fp_.write(self.serial.dumps(sdata)) ret = {'success': False} function_name = data['fun'] if function_name in self.modules.value: try: func = self.modules.value[data['fun']] args, kwargs = salt.minion.load_args_and_kwargs( func, salt.utils.args.parse_input(data['arg'], no_parse=data.get( 'no_parse', [])), data) sys.modules[func.__module__].__context__['retcode'] = 0 executors = data.get('module_executors') or self.opts.get( 'module_executors', ['direct_call']) if isinstance(executors, six.string_types): executors = [executors] elif not isinstance(executors, list) or not executors: raise SaltInvocationError( 'Wrong executors specification: {0}. String or ' 'non-empty list expected'.format(executors)) if self.opts.get('sudo_user', '') and executors[-1] != 'sudo': executors[-1] = 'sudo.get' # replace log.trace("Executors list %s", executors) for name in executors: if name not in self.module_executors.value: raise SaltInvocationError( "Executor '{0}' is not available".format(name)) return_data = self.module_executors.value[name].execute( self.opts, data, func, args, kwargs) if return_data is not None: break if isinstance(return_data, types.GeneratorType): ind = 0 iret = {} for single in return_data: if isinstance(single, dict) and isinstance(iret, list): iret.update(single) else: if not iret: iret = [] iret.append(single) tag = tagify([ data['jid'], 'prog', self.opts['id'], six.text_type(ind) ], 'job') event_data = {'return': single} self._fire_master(event_data, tag) # Need to look into this ind += 1 ret['return'] = iret else: ret['return'] = return_data ret['retcode'] = sys.modules[func.__module__].__context__.get( 'retcode', 0) ret['success'] = True except CommandNotFoundError as exc: msg = 'Command required for \'{0}\' not found'.format( function_name) log.debug(msg, exc_info=True) ret['return'] = '{0}: {1}'.format(msg, exc) except CommandExecutionError as exc: log.error('A command in \'%s\' had a problem: %s', function_name, exc, exc_info_on_loglevel=logging.DEBUG) ret['return'] = 'ERROR: {0}'.format(exc) except SaltInvocationError as exc: log.error('Problem executing \'%s\': %s', function_name, exc, exc_info_on_loglevel=logging.DEBUG) ret['return'] = 'ERROR executing \'{0}\': {1}'.format( function_name, exc) except TypeError as exc: msg = ('TypeError encountered executing {0}: {1}. See ' 'debug log for more info.').format(function_name, exc) log.warning(msg, exc_info_on_loglevel=logging.DEBUG) ret['return'] = msg except Exception: msg = 'The minion function caused an exception' log.warning(msg, exc_info_on_loglevel=logging.DEBUG) ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc()) else: ret['return'] = '\'{0}\' is not available.'.format(function_name) ret['jid'] = data['jid'] ret['fun'] = data['fun'] ret['fun_args'] = data['arg'] self._return_pub(msg, ret, stack) if data['ret']: ret['id'] = self.opts['id'] for returner in set(data['ret'].split(',')): try: self.returners.value['{0}.returner'.format(returner)](ret) except Exception as exc: log.error('The return failed for job %s %s', data['jid'], exc) console.concise("Closing Jobber Stack {0}\n".format(stack.name)) stack.server.close() salt.transport.jobber_stack = None
def update(): ''' When we are asked to update (regular interval) lets reap the cache ''' try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'roots/hash'), find_file) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass mtime_map_path = os.path.join(__opts__['cachedir'], 'roots/mtime_map') # data to send on event data = {'changed': False, 'files': {'changed': []}, 'backend': 'roots'} # generate the new map new_mtime_map = salt.fileserver.generate_mtime_map(__opts__, __opts__['file_roots']) old_mtime_map = {} # if you have an old map, load that if os.path.exists(mtime_map_path): with salt.utils.fopen(mtime_map_path, 'r') as fp_: for line in fp_: try: file_path, mtime = line.replace('\n', '').split(':', 1) old_mtime_map[file_path] = mtime if mtime != str(new_mtime_map.get(file_path, mtime)): data['files']['changed'].append(file_path) except ValueError: # Document the invalid entry in the log log.warning( 'Skipped invalid cache mtime entry in {0}: {1}'.format( mtime_map_path, line)) # compare the maps, set changed to the return value data['changed'] = salt.fileserver.diff_mtime_map(old_mtime_map, new_mtime_map) # compute files that were removed and added old_files = set(old_mtime_map.keys()) new_files = set(new_mtime_map.keys()) data['files']['removed'] = list(old_files - new_files) data['files']['added'] = list(new_files - old_files) # write out the new map mtime_map_path_dir = os.path.dirname(mtime_map_path) if not os.path.exists(mtime_map_path_dir): os.makedirs(mtime_map_path_dir) with salt.utils.fopen(mtime_map_path, 'w') as fp_: for file_path, mtime in six.iteritems(new_mtime_map): fp_.write('{file_path}:{mtime}\n'.format(file_path=file_path, mtime=mtime)) if __opts__.get('fileserver_events', False): # if there is a change, fire an event event = salt.utils.event.get_event('master', __opts__['sock_dir'], __opts__['transport'], opts=__opts__, listen=False) event.fire_event(data, tagify(['roots', 'update'], prefix='fileserver'))
def testPresenceAvailable(self): """ Test Presenter 'available' request (A1, B*) """ console.terse("{0}\n".format(self.testPresenceAvailable.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMaster") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("PresenterTestSetup") act = self.addRecurDeed("SaltRaetPresenter") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add available minions self.addAvailable('alpha') self.addAvailable('beta') self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234') self.addPresenceInfo('aliveds', 'beta', '1.2.3.4', '1234') # add presence request testStack = self.store.fetch('.salt.test.lane.stack').value presenceReq = self.store.fetch('.salt.presence.event_req').value ryn = 'manor' # general available request format presenceReq.append({ 'route': { 'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None) }, 'data': { 'state': 'available' } }) # missing 'data', fallback to available presenceReq.append({ 'route': { 'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None) } }) # missing 'state' in 'data', fallback to available presenceReq.append({ 'route': { 'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None) }, 'data': {} }) # requested None state, fallback to available presenceReq.append({ 'route': { 'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None) }, 'data': { 'state': None } }) # requested 'present' state that is alias for available presenceReq.append({ 'route': { 'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None) }, 'data': { 'state': 'present' } }) # Test # process 5 requests at once self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 5) tag = tagify('present', 'presence') while testStack.rxMsgs: msg, sender = testStack.rxMsgs.popleft() self.assertDictEqual( msg, { 'route': { 'src': [None, 'manor', None], 'dst': [None, None, 'event_fire'] }, 'tag': tag, 'data': { 'present': { 'alpha': '1.1.1.1', 'beta': '1.2.3.4' } } }) # Close active stacks servers act.actor.lane_stack.value.server.close() testStack = self.store.fetch('.salt.test.lane.stack') if testStack: testStack.value.server.close()
def cmd(self, fun, arg, pub_data=None, kwarg=None): ''' Execute a runner function .. code-block:: python >>> opts = salt.config.master_config('/etc/salt/master') >>> runner = salt.runner.RunnerClient(opts) >>> runner.cmd('jobs.list_jobs', []) { '20131219215650131543': { 'Arguments': [300], 'Function': 'test.sleep', 'StartTime': '2013, Dec 19 21:56:50.131543', 'Target': '*', 'Target-type': 'glob', 'User': '******' }, '20131219215921857715': { 'Arguments': [300], 'Function': 'test.sleep', 'StartTime': '2013, Dec 19 21:59:21.857715', 'Target': '*', 'Target-type': 'glob', 'User': '******' }, } ''' if kwarg is None: kwarg = {} if not isinstance(kwarg, dict): raise salt.exceptions.SaltInvocationError( 'kwarg must be formatted as a dictionary') if pub_data is None: pub_data = {} if not isinstance(pub_data, dict): raise salt.exceptions.SaltInvocationError( 'pub_data must be formatted as a dictionary') arglist = salt.utils.args.parse_input(arg) def _append_kwarg(arglist, kwarg): ''' Append the kwarg dict to the arglist ''' kwarg['__kwarg__'] = True arglist.append(kwarg) if kwarg: try: if isinstance(arglist[-1], dict) \ and '__kwarg__' in arglist[-1]: for key, val in six.iteritems(kwarg): if key in arglist[-1]: log.warning( 'Overriding keyword argument {0!r}'.format( key)) arglist[-1][key] = val else: # No kwargs yet present in arglist _append_kwarg(arglist, kwarg) except IndexError: # arglist is empty, just append _append_kwarg(arglist, kwarg) self._verify_fun(fun) args, kwargs = salt.minion.load_args_and_kwargs( self.functions[fun], arglist, pub_data) fstr = '{0}.prep_jid'.format(self.opts['master_job_cache']) jid = self.returners[fstr]() log.debug('Runner starting with jid {0}'.format(jid)) self.event.fire_event({'runner_job': fun}, tagify([jid, 'new'], 'job')) target = RunnerClient._thread_return data = {'fun': fun, 'jid': jid, 'args': args, 'kwargs': kwargs} args = (self, self.opts, data) ret = jid if self.opts.get('async', False): process = multiprocessing.Process(target=target, args=args) process.start() else: ret = target(*args) return ret
def testPresenceAllowedOneMinion(self): """ Test Presenter 'allowed' request with one minion in the state (D5) """ console.terse("{0}\n".format( self.testPresenceAllowedOneMinion.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMaster") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("PresenterTestSetup") act = self.addRecurDeed("SaltRaetPresenter") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add allowed minions self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234') # add presence request testStack = self.store.fetch('.salt.test.lane.stack').value presenceReq = self.store.fetch('.salt.presence.event_req').value ryn = 'manor' msg = { 'route': { 'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None) }, 'data': { 'state': 'allowed' } } presenceReq.append(msg) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 1) tag = tagify('present', 'presence') msg, sender = testStack.rxMsgs.popleft() self.assertDictEqual( msg, { 'route': { 'src': [None, 'manor', None], 'dst': [None, None, 'event_fire'] }, 'tag': tag, 'data': { 'allowed': { 'alpha': '1.1.1.1' } } }) # Close active stacks servers act.actor.lane_stack.value.server.close() testStack = self.store.fetch('.salt.test.lane.stack') if testStack: testStack.value.server.close()
def update(): ''' Execute an hg pull on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'hgfs'} # _clear_old_remotes runs init(), so use the value from there to avoid a # second init() data['changed'], repos = _clear_old_remotes() for repo in repos: if os.path.exists(repo['lockfile']): log.warning( 'Update lockfile is present for hgfs remote {0}, skipping. ' 'If this warning persists, it is possible that the update ' 'process was interrupted. Removing {1} or running ' '\'salt-run fileserver.clear_lock hgfs\' will allow updates ' 'to continue for this remote.'.format(repo['url'], repo['lockfile'])) continue _, errors = lock(repo) if errors: log.error('Unable to set update lock for hgfs remote {0}, ' 'skipping.'.format(repo['url'])) continue log.debug('hgfs is fetching from {0}'.format(repo['url'])) repo['repo'].open() curtip = repo['repo'].tip() try: repo['repo'].pull() except Exception as exc: log.error( 'Exception {0} caught while updating hgfs remote {1}'.format( exc, repo['url']), exc_info_on_loglevel=logging.DEBUG) else: newtip = repo['repo'].tip() if curtip[1] != newtip[1]: data['changed'] = True repo['repo'].close() clear_lock(repo) env_cache = os.path.join(__opts__['cachedir'], 'hgfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): env_cachedir = os.path.dirname(env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.fopen(env_cache, 'w+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) # if there is a change, fire an event if __opts__.get('fileserver_events', False): event = salt.utils.event.get_event('master', __opts__['sock_dir'], __opts__['transport'], opts=__opts__, listen=False) event.fire_event(data, tagify(['hgfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'hgfs/hash'), find_file) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def update(): ''' Execute a git fetch on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'gitfs'} provider = _get_provider() pid = os.getpid() data['changed'] = purge_cache() for repo_conf in init(): repo = repo_conf['repo'] if provider == 'gitpython': origin = repo.remotes[0] working_dir = repo.working_dir elif provider == 'pygit2': origin = repo.remotes[0] working_dir = repo.workdir elif provider == 'dulwich': # origin is just a uri here, there is no origin object origin = repo_conf['uri'] working_dir = repo.path lk_fn = os.path.join(working_dir, 'update.lk') with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write(str(pid)) try: if provider == 'gitpython': for fetch in origin.fetch(): if fetch.old_commit is not None: data['changed'] = True elif provider == 'pygit2': fetch = origin.fetch() if fetch.get('received_objects', 0): data['changed'] = True elif provider == 'dulwich': client, path = \ dulwich.client.get_transport_and_path_from_url( origin, thin_packs=True ) refs_pre = repo.get_refs() try: refs_post = client.fetch(path, repo) except KeyError: log.critical( 'Local repository cachedir {0!r} (corresponding ' 'remote: {1}) has been corrupted. Salt will now ' 'attempt to remove the local checkout to allow it to ' 'be re-initialized in the next fileserver cache ' 'update.' .format(repo_conf['cachedir'], repo_conf['uri']) ) try: salt.utils.rm_rf(repo_conf['cachedir']) except OSError as exc: log.critical( 'Unable to remove {0!r}: {1}' .format(repo_conf['cachedir'], exc) ) continue if refs_post is None: # Empty repository log.warning( 'gitfs remote {0!r} is an empty repository and will ' 'be skipped.'.format(origin) ) continue if refs_pre != refs_post: data['changed'] = True # Update local refs for ref in _dulwich_env_refs(refs_post): repo[ref] = refs_post[ref] # Prune stale refs for ref in repo.get_refs(): if ref not in refs_post: del repo[ref] except Exception as exc: log.warning( 'Exception caught while fetching: {0}'.format(exc) ) try: os.remove(lk_fn) except (IOError, OSError): pass env_cache = os.path.join(__opts__['cachedir'], 'gitfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.fopen(env_cache, 'w+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) # if there is a change, fire an event if __opts__.get('fileserver_events', False): event = salt.utils.event.MasterEvent(__opts__['sock_dir']) event.fire_event(data, tagify(['gitfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'gitfs/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def update(): """ Execute an hg pull on all of the repos """ # data for the fileserver event data = {"changed": False, "backend": "hgfs"} # _clear_old_remotes runs init(), so use the value from there to avoid a # second init() data["changed"], repos = _clear_old_remotes() for repo in repos: if os.path.exists(repo["lockfile"]): log.warning( "Update lockfile is present for hgfs remote %s, skipping. " "If this warning persists, it is possible that the update " "process was interrupted. Removing %s or running " "'salt-run fileserver.clear_lock hgfs' will allow updates " "to continue for this remote.", repo["url"], repo["lockfile"], ) continue _, errors = lock(repo) if errors: log.error( "Unable to set update lock for hgfs remote %s, skipping.", repo["url"] ) continue log.debug("hgfs is fetching from %s", repo["url"]) repo["repo"].open() curtip = repo["repo"].tip() try: repo["repo"].pull() except Exception as exc: # pylint: disable=broad-except log.error( "Exception %s caught while updating hgfs remote %s", exc, repo["url"], exc_info_on_loglevel=logging.DEBUG, ) else: newtip = repo["repo"].tip() if curtip[1] != newtip[1]: data["changed"] = True repo["repo"].close() clear_lock(repo) env_cache = os.path.join(__opts__["cachedir"], "hgfs/envs.p") if data.get("changed", False) is True or not os.path.isfile(env_cache): env_cachedir = os.path.dirname(env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) with salt.utils.files.fopen(env_cache, "wb+") as fp_: fp_.write(salt.payload.dumps(new_envs)) log.trace("Wrote env cache data to %s", env_cache) # if there is a change, fire an event if __opts__.get("fileserver_events", False): with salt.utils.event.get_event( "master", __opts__["sock_dir"], __opts__["transport"], opts=__opts__, listen=False, ) as event: event.fire_event(data, tagify(["hgfs", "update"], prefix="fileserver")) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__["cachedir"], "hgfs/hash"), find_file ) except OSError: # Hash file won't exist if no files have yet been served up pass
def _gen_async_pub(self, jid=None): if jid is None: jid = salt.utils.jid.gen_jid() tag = tagify(jid, prefix=self.tag_prefix) return {'tag': tag, 'jid': jid}
def store_job(opts, load, event=None, mminion=None): ''' Store job information using the configured master_job_cache ''' # If the return data is invalid, just ignore it if any(key not in load for key in ('return', 'jid', 'id')): return False if not salt.utils.verify.valid_id(opts, load['id']): return False if mminion is None: mminion = salt.minion.MasterMinion(opts, states=False, rend=False) job_cache = opts['master_job_cache'] if load['jid'] == 'req': # The minion is returning a standalone job, request a jobid load['arg'] = load.get('arg', load.get('fun_args', [])) load['tgt_type'] = 'glob' load['tgt'] = load['id'] prep_fstr = '{0}.prep_jid'.format(opts['master_job_cache']) try: load['jid'] = mminion.returners[prep_fstr]( nocache=load.get('nocache', False)) except KeyError: emsg = "Returner '{0}' does not support function prep_jid".format( job_cache) log.error(emsg) raise KeyError(emsg) # save the load, since we don't have it saveload_fstr = '{0}.save_load'.format(job_cache) try: mminion.returners[saveload_fstr](load['jid'], load) except KeyError: emsg = "Returner '{0}' does not support function save_load".format( job_cache) log.error(emsg) raise KeyError(emsg) elif salt.utils.jid.is_jid(load['jid']): # Store the jid jidstore_fstr = '{0}.prep_jid'.format(job_cache) try: mminion.returners[jidstore_fstr](False, passed_jid=load['jid']) except KeyError: emsg = "Returner '{0}' does not support function prep_jid".format( job_cache) log.error(emsg) raise KeyError(emsg) if event: # If the return data is invalid, just ignore it log.info('Got return from {id} for job {jid}'.format(**load)) event.fire_event(load, tagify([load['jid'], 'ret', load['id']], 'job')) event.fire_ret_load(load) # if you have a job_cache, or an ext_job_cache, don't write to # the regular master cache if not opts['job_cache'] or opts.get('ext_job_cache'): return # otherwise, write to the master cache fstr = '{0}.returner'.format(job_cache) if 'fun' not in load and load.get('return', {}): ret_ = load.get('return', {}) if 'fun' in ret_: load.update({'fun': ret_['fun']}) if 'user' in ret_: load.update({'user': ret_['user']}) try: mminion.returners[fstr](load) except KeyError: emsg = "Returner '{0}' does not support function returner".format( job_cache) log.error(emsg) raise KeyError(emsg)
def low(self, fun, low): ''' Execute a function from low data Low data includes: required: - fun: the name of the function to run optional: - args: a list of args to pass to fun - kwargs: kwargs for fun - __user__: user who is running the command - __jid__: jid to run under - __tag__: tag to run under ''' # fire the mminion loading (if not already done) here # this is not to clutter the output with the module loading # if we have a high debug level. self.mminion # pylint: disable=W0104 jid = low.get('__jid__', salt.utils.jid.gen_jid()) tag = low.get('__tag__', tagify(jid, prefix=self.tag_prefix)) data = {'fun': '{0}.{1}'.format(self.client, fun), 'jid': jid, 'user': low.get('__user__', 'UNKNOWN'), } event = salt.utils.event.get_event( 'master', self.opts['sock_dir'], self.opts['transport'], opts=self.opts, listen=False) namespaced_event = salt.utils.event.NamespacedEvent( event, tag, print_func=self.print_async_event if hasattr(self, 'print_async_event') else None ) # TODO: document these, and test that they exist # TODO: Other things to inject?? func_globals = {'__jid__': jid, '__user__': data['user'], '__tag__': tag, # weak ref to avoid the Exception in interpreter # teardown of event '__jid_event__': weakref.proxy(namespaced_event), } func_globals['__jid_event__'].fire_event(data, 'new') try: verify_fun(self.functions, fun) # Inject some useful globals to *all* the function's global # namespace only once per module-- not per func completed_funcs = [] _functions = copy.deepcopy(self.functions) for mod_name in six.iterkeys(_functions): if '.' not in mod_name: continue mod, _ = mod_name.split('.', 1) if mod in completed_funcs: continue completed_funcs.append(mod) for global_key, value in six.iteritems(func_globals): self.functions[mod_name].__globals__[global_key] = value # There are some descrepencies of what a "low" structure is in the # publisher world it is a dict including stuff such as jid, fun, # arg (a list of args, with kwargs packed in). Historically this # particular one has had no "arg" and just has had all the kwargs # packed into the top level object. The plan is to move away from # that since the caller knows what is an arg vs a kwarg, but while # we make the transition we will load "kwargs" using format_call if # there are no kwargs in the low object passed in f_call = None if 'args' not in low: f_call = salt.utils.format_call( self.functions[fun], low, expected_extra_kws=CLIENT_INTERNAL_KEYWORDS ) args = f_call.get('args', ()) else: args = low['args'] if 'kwargs' not in low: if f_call is None: f_call = salt.utils.format_call( self.functions[fun], low, expected_extra_kws=CLIENT_INTERNAL_KEYWORDS ) kwargs = f_call.get('kwargs', {}) # throw a warning for the badly formed low data if we found # kwargs using the old mechanism if kwargs: salt.utils.warn_until( 'Boron', 'kwargs must be passed inside the low under "kwargs"' ) else: kwargs = low['kwargs'] data['return'] = self.functions[fun](*args, **kwargs) data['success'] = True except (Exception, SystemExit) as ex: if isinstance(ex, salt.exceptions.NotImplemented): data['return'] = str(ex) else: data['return'] = 'Exception occurred in {0} {1}: {2}'.format( self.client, fun, traceback.format_exc(), ) data['success'] = False namespaced_event.fire_event(data, 'ret') try: salt.utils.job.store_job( self.opts, {'id': self.opts['id'], 'tgt': self.opts['id'], 'jid': data['jid'], 'return': data, }, event=None, mminion=self.mminion, ) except salt.exceptions.SaltCacheError: log.error('Could not store job cache info. Job details for this run may be unavailable.') # if we fired an event, make sure to delete the event object. # This will ensure that we call destroy, which will do the 0MQ linger log.info('Runner completed: {0}'.format(data['jid'])) del event del namespaced_event return data['return']
def testMinionLaneStats(self): ''' Test Minion Road Stats request (A2) ''' console.terse("{0}\n".format(self.testMinionLaneStats.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMinion") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("SaltRaetRoadStackSetup") self.addEnterDeed("StatsMinionTestSetup") act = self.addRecurDeed("SaltRaetStatsEventerMinion") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add a test stat key-value roadStack = self.store.fetch('.salt.road.manor.stack') laneStack = self.store.fetch('.salt.lane.manor.stack') roadStack.value.stats = odict() laneStack.value.stats = odict({'test_stats_event': 111}) # ensure stats are equal to expected self.assertDictEqual(roadStack.value.stats, {}) self.assertDictEqual(laneStack.value.stats, {'test_stats_event': 111}) # add stats request testStack = self.store.fetch('.salt.test.road.stack').value statsReq = self.store.fetch('.salt.stats.event_req').value tag = tagify('lane', 'stats') minionName = roadStack.value.local.name masterName = testStack.local.name # lane stats request statsReq.append({ 'route': { 'dst': (minionName, None, 'stats_req'), 'src': (masterName, None, None) }, 'tag': tag }) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 1) msg, sender = testStack.rxMsgs.popleft() self.assertDictEqual( msg, { 'route': { 'src': [ns2u(minionName), 'manor', None], 'dst': [ns2u(masterName), None, 'event_fire'] }, 'tag': ns2u(tag), 'data': { 'test_stats_event': 111 } }) # Close active stacks servers act.actor.lane_stack.value.server.close() act.actor.road_stack.value.server.close() testStack = self.store.fetch('.salt.test.road.stack') if testStack: testStack.value.server.close()
def update(): ''' Execute an svn update on all of the repos ''' # data for the fileserver event data = {'changed': False, 'backend': 'svnfs'} # _clear_old_remotes runs init(), so use the value from there to avoid a # second init() data['changed'], repos = _clear_old_remotes() for repo in repos: if os.path.exists(repo['lockfile']): log.warning( 'Update lockfile is present for svnfs remote %s, skipping. ' 'If this warning persists, it is possible that the update ' 'process was interrupted. Removing %s or running ' '\'salt-run fileserver.clear_lock svnfs\' will allow updates ' 'to continue for this remote.', repo['url'], repo['lockfile'] ) continue _, errors = lock(repo) if errors: log.error( 'Unable to set update lock for svnfs remote %s, skipping.', repo['url'] ) continue log.debug('svnfs is fetching from %s', repo['url']) old_rev = _rev(repo) try: CLIENT.update(repo['repo']) except pysvn._pysvn.ClientError as exc: log.error( 'Error updating svnfs remote %s (cachedir: %s): %s', repo['url'], repo['cachedir'], exc ) new_rev = _rev(repo) if any((x is None for x in (old_rev, new_rev))): # There were problems getting the revision ID continue if new_rev != old_rev: data['changed'] = True clear_lock(repo) env_cache = os.path.join(__opts__['cachedir'], 'svnfs/envs.p') if data.get('changed', False) is True or not os.path.isfile(env_cache): env_cachedir = os.path.dirname(env_cache) if not os.path.exists(env_cachedir): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) with salt.utils.files.fopen(env_cache, 'wb+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to %s', env_cache) # if there is a change, fire an event if __opts__.get('fileserver_events', False): with salt.utils.event.get_event( 'master', __opts__['sock_dir'], __opts__['transport'], opts=__opts__, listen=False) as event: event.fire_event(data, tagify(['svnfs', 'update'], prefix='fileserver')) try: salt.fileserver.reap_fileserver_cache_dir( os.path.join(__opts__['cachedir'], 'svnfs/hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass
def thread_return(cls, minion_instance, opts, data): """ This method should be used as a threading target, start the actual minion side execution. """ fn_ = os.path.join(minion_instance.proc_dir, data["jid"]) salt.utils.process.appendproctitle("{}._thread_return {}".format( cls.__name__, data["jid"])) sdata = {"pid": os.getpid()} sdata.update(data) log.info("Starting a new job with PID %s", sdata["pid"]) with salt.utils.files.fopen(fn_, "w+b") as fp_: fp_.write(minion_instance.serial.dumps(sdata)) ret = {"success": False} function_name = data["fun"] executors = (data.get("module_executors") or getattr(minion_instance, "module_executors", []) or opts.get("module_executors", ["direct_call"])) allow_missing_funcs = any([ minion_instance.executors["{}.allow_missing_func".format(executor)]( function_name) for executor in executors if "{}.allow_missing_func".format(executor) in minion_instance.executors ]) if function_name in minion_instance.functions or allow_missing_funcs is True: try: minion_blackout_violation = False if minion_instance.connected and minion_instance.opts[ "pillar"].get("minion_blackout", False): whitelist = minion_instance.opts["pillar"].get( "minion_blackout_whitelist", []) # this minion is blacked out. Only allow saltutil.refresh_pillar and the whitelist if (function_name != "saltutil.refresh_pillar" and function_name not in whitelist): minion_blackout_violation = True # use minion_blackout_whitelist from grains if it exists if minion_instance.opts["grains"].get("minion_blackout", False): whitelist = minion_instance.opts["grains"].get( "minion_blackout_whitelist", []) if (function_name != "saltutil.refresh_pillar" and function_name not in whitelist): minion_blackout_violation = True if minion_blackout_violation: raise SaltInvocationError( "Minion in blackout mode. Set 'minion_blackout' " "to False in pillar or grains to resume operations. Only " "saltutil.refresh_pillar allowed in blackout mode.") if function_name in minion_instance.functions: func = minion_instance.functions[function_name] args, kwargs = salt.minion.load_args_and_kwargs( func, data["arg"], data) else: # only run if function_name is not in minion_instance.functions and allow_missing_funcs is True func = function_name args, kwargs = data["arg"], data minion_instance.functions.pack["__context__"]["retcode"] = 0 if isinstance(executors, str): executors = [executors] elif not isinstance(executors, list) or not executors: raise SaltInvocationError( "Wrong executors specification: {}. String or non-empty list" " expected".format(executors)) if opts.get("sudo_user", "") and executors[-1] != "sudo": executors[-1] = "sudo" # replace the last one with sudo log.trace("Executors list %s", executors) # pylint: disable=no-member for name in executors: fname = "{}.execute".format(name) if fname not in minion_instance.executors: raise SaltInvocationError( "Executor '{}' is not available".format(name)) return_data = minion_instance.executors[fname](opts, data, func, args, kwargs) if return_data is not None: break if isinstance(return_data, types.GeneratorType): ind = 0 iret = {} for single in return_data: if isinstance(single, dict) and isinstance(iret, dict): iret.update(single) else: if not iret: iret = [] iret.append(single) tag = tagify([data["jid"], "prog", opts["id"], str(ind)], "job") event_data = {"return": single} minion_instance._fire_master(event_data, tag) ind += 1 ret["return"] = iret else: ret["return"] = return_data retcode = minion_instance.functions.pack["__context__"].get( "retcode", salt.defaults.exitcodes.EX_OK) if retcode == salt.defaults.exitcodes.EX_OK: # No nonzero retcode in __context__ dunder. Check if return # is a dictionary with a "result" or "success" key. try: func_result = all( return_data.get(x, True) for x in ("result", "success")) except Exception: # pylint: disable=broad-except # return data is not a dict func_result = True if not func_result: retcode = salt.defaults.exitcodes.EX_GENERIC ret["retcode"] = retcode ret["success"] = retcode == salt.defaults.exitcodes.EX_OK except CommandNotFoundError as exc: msg = "Command required for '{}' not found".format(function_name) log.debug(msg, exc_info=True) ret["return"] = "{}: {}".format(msg, exc) ret["out"] = "nested" ret["retcode"] = salt.defaults.exitcodes.EX_GENERIC except CommandExecutionError as exc: log.error( "A command in '%s' had a problem: %s", function_name, exc, exc_info_on_loglevel=logging.DEBUG, ) ret["return"] = "ERROR: {}".format(exc) ret["out"] = "nested" ret["retcode"] = salt.defaults.exitcodes.EX_GENERIC except SaltInvocationError as exc: log.error( "Problem executing '%s': %s", function_name, exc, exc_info_on_loglevel=logging.DEBUG, ) ret["return"] = "ERROR executing '{}': {}".format( function_name, exc) ret["out"] = "nested" ret["retcode"] = salt.defaults.exitcodes.EX_GENERIC except TypeError as exc: msg = "Passed invalid arguments to {}: {}\n{}".format( function_name, exc, func.__doc__ or "") log.warning(msg, exc_info_on_loglevel=logging.DEBUG) ret["return"] = msg ret["out"] = "nested" ret["retcode"] = salt.defaults.exitcodes.EX_GENERIC except Exception: # pylint: disable=broad-except msg = "The minion function caused an exception" log.warning(msg, exc_info=True) salt.utils.error.fire_exception(salt.exceptions.MinionError(msg), opts, job=data) ret["return"] = "{}: {}".format(msg, traceback.format_exc()) ret["out"] = "nested" ret["retcode"] = salt.defaults.exitcodes.EX_GENERIC else: docs = minion_instance.functions["sys.doc"]( "{}*".format(function_name)) if docs: docs[function_name] = minion_instance.functions.missing_fun_string( function_name) ret["return"] = docs else: ret["return"] = minion_instance.functions.missing_fun_string( function_name) mod_name = function_name.split(".")[0] if mod_name in minion_instance.function_errors: ret["return"] += " Possible reasons: '{}'".format( minion_instance.function_errors[mod_name]) ret["success"] = False ret["retcode"] = salt.defaults.exitcodes.EX_GENERIC ret["out"] = "nested" ret["jid"] = data["jid"] ret["fun"] = data["fun"] ret["fun_args"] = data["arg"] if "master_id" in data: ret["master_id"] = data["master_id"] if "metadata" in data: if isinstance(data["metadata"], dict): ret["metadata"] = data["metadata"] else: log.warning( "The metadata parameter must be a dictionary. Ignoring.") if minion_instance.connected: minion_instance._return_pub( ret, timeout=minion_instance._return_retry_timer()) # Add default returners from minion config # Should have been coverted to comma-delimited string already if isinstance(opts.get("return"), str): if data["ret"]: data["ret"] = ",".join((data["ret"], opts["return"])) else: data["ret"] = opts["return"] log.debug("minion return: %s", ret) # TODO: make a list? Seems odd to split it this late :/ if data["ret"] and isinstance(data["ret"], str): if "ret_config" in data: ret["ret_config"] = data["ret_config"] if "ret_kwargs" in data: ret["ret_kwargs"] = data["ret_kwargs"] ret["id"] = opts["id"] for returner in set(data["ret"].split(",")): try: returner_str = "{}.returner".format(returner) if returner_str in minion_instance.returners: minion_instance.returners[returner_str](ret) else: returner_err = minion_instance.returners.missing_fun_string( returner_str) log.error( "Returner %s could not be loaded: %s", returner_str, returner_err, ) except Exception as exc: # pylint: disable=broad-except log.exception("The return failed for job %s: %s", data["jid"], exc)
def proc_run(self, msg): ''' Execute the run in a dedicated process ''' data = msg['pub'] fn_ = os.path.join(self.proc_dir, data['jid']) self.opts['__ex_id'] = data['jid'] salt.utils.daemonize_if(self.opts) salt.transport.jobber_stack = stack = self._setup_jobber_stack() # set up return destination from source src_estate, src_yard, src_share = msg['route']['src'] salt.transport.jobber_estate_name = src_estate salt.transport.jobber_yard_name = src_yard sdata = {'pid': os.getpid()} sdata.update(data) with salt.utils.fopen(fn_, 'w+') as fp_: fp_.write(self.serial.dumps(sdata)) ret = {'success': False} function_name = data['fun'] if function_name in self.modules.value: try: func = self.modules.value[data['fun']] args, kwargs = salt.minion.load_args_and_kwargs( func, salt.utils.args.parse_input(data['arg']), data) sys.modules[func.__module__].__context__['retcode'] = 0 return_data = func(*args, **kwargs) if isinstance(return_data, types.GeneratorType): ind = 0 iret = {} for single in return_data: if isinstance(single, dict) and isinstance(iret, list): iret.update(single) else: if not iret: iret = [] iret.append(single) tag = tagify( [data['jid'], 'prog', self.opts['id'], str(ind)], 'job') event_data = {'return': single} self._fire_master(event_data, tag) # Need to look into this ind += 1 ret['return'] = iret else: ret['return'] = return_data ret['retcode'] = sys.modules[func.__module__].__context__.get( 'retcode', 0) ret['success'] = True except CommandNotFoundError as exc: msg = 'Command required for {0!r} not found'.format( function_name) log.debug(msg, exc_info=True) ret['return'] = '{0}: {1}'.format(msg, exc) except CommandExecutionError as exc: log.error('A command in {0!r} had a problem: {1}'.format( function_name, exc), exc_info_on_loglevel=logging.DEBUG) ret['return'] = 'ERROR: {0}'.format(exc) except SaltInvocationError as exc: log.error('Problem executing {0!r}: {1}'.format( function_name, exc), exc_info_on_loglevel=logging.DEBUG) ret['return'] = 'ERROR executing {0!r}: {1}'.format( function_name, exc) except TypeError as exc: msg = ('TypeError encountered executing {0}: {1}. See ' 'debug log for more info.').format(function_name, exc) log.warning(msg, exc_info_on_loglevel=logging.DEBUG) ret['return'] = msg except Exception: msg = 'The minion function caused an exception' log.warning(msg, exc_info_on_loglevel=logging.DEBUG) ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc()) else: ret['return'] = '{0!r} is not available.'.format(function_name) ret['jid'] = data['jid'] ret['fun'] = data['fun'] ret['fun_args'] = data['arg'] self._return_pub(msg, ret, stack) if data['ret']: ret['id'] = self.opts['id'] for returner in set(data['ret'].split(',')): try: self.returners.value['{0}.returner'.format(returner)](ret) except Exception as exc: log.error('The return failed for job {0} {1}'.format( data['jid'], exc)) console.concise("Closing Jobber Stack {0}\n".format(stack.name)) stack.server.close() salt.transport.jobber_stack = None
def testPresenceAvailableSomeIpUnknown(self): ''' Test Presenter 'available' request with some minion addresses aren't known (D3) ''' console.terse("{0}\n".format( self.testPresenceAvailableSomeIpUnknown.__doc__)) # Bootstrap self.addEnterDeed("TestOptsSetupMaster") self.addEnterDeed("SaltRaetManorLaneSetup") self.addEnterDeed("PresenterTestSetup") act = self.addRecurDeed("SaltRaetPresenter") self.resolve() # resolve House, Framer, Frame, Acts, Actors self.frame.enter() # Prepare # add available minions self.addAvailable('alpha') self.addAvailable('beta') self.addAvailable('gamma') self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234') self.addPresenceInfo('aliveds', 'delta', '1.2.3.4', '1234') # add presence request testStack = self.store.fetch('.salt.test.lane.stack').value presenceReq = self.store.fetch('.salt.presence.event_req').value ryn = 'manor' presenceReq.append({ 'route': { 'dst': (None, ryn, 'presence_req'), 'src': (None, testStack.local.name, None) }, 'data': { 'state': 'available' } }) # Test self.frame.recur() # run in frame # Check self.assertEqual(len(testStack.rxMsgs), 0) testStack.serviceAll() self.assertEqual(len(testStack.rxMsgs), 1) tag = tagify('present', 'presence') msg, sender = testStack.rxMsgs.popleft() self.assertDictEqual( msg, { 'route': { 'src': [None, 'manor', None], 'dst': [None, None, 'event_fire'] }, 'tag': tag, 'data': { 'present': { 'alpha': '1.1.1.1', 'beta': None, 'gamma': None } } }) # Close active stacks servers act.actor.lane_stack.value.server.close() testStack = self.store.fetch('.salt.test.lane.stack') if testStack: testStack.value.server.close()
def low(self, fun, low): ''' Execute a function from low data Low data includes: required: - fun: the name of the function to run optional: - args: a list of args to pass to fun - kwargs: kwargs for fun - __user__: user who is running the command - __jid__: jid to run under - __tag__: tag to run under ''' jid = low.get('__jid__', salt.utils.jid.gen_jid()) tag = low.get('__tag__', tagify(jid, prefix=self.tag_prefix)) data = {'fun': '{0}.{1}'.format(self.client, fun), 'jid': jid, 'user': low.get('__user__', 'UNKNOWN'), } event = salt.utils.event.get_event( 'master', self.opts['sock_dir'], self.opts['transport'], opts=self.opts, listen=False) event.fire_event(data, tagify('new', base=tag)) # TODO: document these, and test that they exist # TODO: Other things to inject?? func_globals = {'__jid__': jid, '__user__': data['user'], '__tag__': tag, '__jid_event__': salt.utils.event.NamespacedEvent(event, tag), } def over_print(output): ''' Print and duplicate the print to an event ''' print_event = {'data': output, 'outputter': 'pprint'} func_globals['__jid_event__'].fire_event(print_event, 'print') __builtin__.print(output) # and do the old style printout func_globals['print'] = over_print # Inject some useful globals to the funciton's global namespace for global_key, value in func_globals.iteritems(): self.functions[fun].func_globals[global_key] = value try: self._verify_fun(fun) # There are some descrepencies of what a "low" structure is # in the publisher world it is a dict including stuff such as jid, # fun, arg (a list of args, with kwargs packed in). Historically # this particular one has had no "arg" and just has had all the # kwargs packed into the top level object. The plan is to move away # from that since the caller knows what is an arg vs a kwarg, but # while we make the transition we will load "kwargs" using format_call # if there are no kwargs in the low object passed in f_call = None if 'args' not in low: f_call = salt.utils.format_call(self.functions[fun], low) args = f_call.get('args', ()) else: args = low['args'] if 'kwargs' not in low: if f_call is None: f_call = salt.utils.format_call(self.functions[fun], low) kwargs = f_call.get('kwargs', {}) # throw a warning for the badly formed low data if we found # kwargs using the old mechanism if kwargs: salt.utils.warn_until( 'Boron', 'kwargs must be passed inside the low under "kwargs"' ) else: kwargs = low['kwargs'] data['return'] = self.functions[fun](*args, **kwargs) data['success'] = True except Exception as exc: data['return'] = 'Exception occurred in {0} {1}: {2}: {3}'.format( self.client, fun, exc.__class__.__name__, exc, ) data['success'] = False event.fire_event(data, tagify('ret', base=tag)) # if we fired an event, make sure to delete the event object. # This will ensure that we call destroy, which will do the 0MQ linger del event return data['return']