class RAETCaller(BaseCaller):
'''
Object to wrap the calling of local salt modules for the salt-call command
when transport is raet
There are two operation modes.
1) Use a preexisting minion
2) Set up a special caller minion if no preexisting minion
The special caller minion is a subset whose only function is to perform
Salt-calls with raet as the transport
The essentials:
A RoadStack whose local estate name is of the form "role_kind" where:
role is the minion id opts['id']
kind is opts['__role'] which should be 'caller' APPL_KIND_NAMES
The RoadStack if for communication to/from a master
A LaneStack with manor yard so that RaetChannels created by the func Jobbers
can communicate through this manor yard then through the
RoadStack to/from a master
A Router to route between the stacks (Road and Lane)
These are all managed via a FloScript named caller.flo
'''
def __init__(self, opts):
'''
Pass in the command line options
'''
self.process = None
if not opts['local']:
self.stack = self._setup_caller_stack(opts)
salt.transport.jobber_stack = self.stack
if (opts.get('__role') ==
kinds.APPL_KIND_NAMES[kinds.applKinds.caller]):
# spin up and fork minion here
self.process = MultiprocessingProcess(target=raet_minion_run,
kwargs={'cleanup_protecteds': [self.stack.ha], })
self.process.start()
# wait here until '/var/run/salt/minion/alpha_caller.manor.uxd' exists
self._wait_caller(opts)
super(RAETCaller, self).__init__(opts)
def run(self):
'''
Execute the salt call logic
'''
try:
ret = self.call()
if not self.opts['local']:
self.stack.server.close()
salt.transport.jobber_stack = None
if self.opts['print_metadata']:
print_ret = ret
else:
print_ret = ret.get('return', {})
if self.process:
self.process.terminate()
salt.output.display_output(
{'local': print_ret},
ret.get('out', 'nested'),
self.opts)
if self.opts.get('retcode_passthrough', False):
sys.exit(ret['retcode'])
except SaltInvocationError as err:
raise SystemExit(err)
def _setup_caller_stack(self, opts):
'''
Setup and return the LaneStack and Yard used by by channel when global
not already setup such as in salt-call to communicate to-from the minion
'''
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup RAETChannel.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAETChannel.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAETChannel.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = opts['sock_dir']
stackname = 'caller' + nacling.uuid(size=18)
stack = LaneStack(name=stackname,
lanename=lanename,
sockdirpath=sockdirpath)
stack.Pk = raeting.PackKind.pack.value
stack.addRemote(RemoteYard(stack=stack,
name='manor',
lanename=lanename,
dirpath=sockdirpath))
log.debug("Created Caller Jobber Stack {0}\n".format(stack.name))
return stack
def _wait_caller(self, opts):
'''
Returns when RAET Minion Yard is available
'''
yardname = 'manor'
dirpath = opts['sock_dir']
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup RAET SaltCaller.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAET SaltCaller.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAET SaltCaller.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
ha, dirpath = Yard.computeHa(dirpath, lanename, yardname)
if is_windows():
# RAET lanes do not use files on Windows. Need to use win32file
# API to check for existence.
exists = False
while not exists:
try:
f = win32file.CreateFile(
ha,
win32file.GENERIC_WRITE | win32file.GENERIC_READ,
win32file.FILE_SHARE_READ,
None,
win32file.OPEN_EXISTING,
0,
None)
win32file.CloseHandle(f)
exists = True
except win32file.error:
time.sleep(0.1)
else:
while not ((os.path.exists(ha) and
not os.path.isfile(ha) and
not os.path.isdir(ha))):
time.sleep(0.1)
time.sleep(0.5)
def handle_ssh(self, mine=False):
'''
Spin up the needed threads or processes and execute the subsequent
routines
'''
que = multiprocessing.Queue()
running = {}
target_iter = self.targets.__iter__()
returned = set()
rets = set()
init = False
while True:
if not self.targets:
log.error('No matching targets found in roster.')
break
if len(running) < self.opts.get('ssh_max_procs', 25) and not init:
try:
host = next(target_iter)
except StopIteration:
init = True
continue
for default in self.defaults:
if default not in self.targets[host]:
self.targets[host][default] = self.defaults[default]
args = (
que,
self.opts,
host,
self.targets[host],
mine,
)
routine = MultiprocessingProcess(
target=self.handle_routine,
args=args)
routine.start()
running[host] = {'thread': routine}
continue
ret = {}
try:
ret = que.get(False)
if 'id' in ret:
returned.add(ret['id'])
yield {ret['id']: ret['ret']}
except Exception:
# This bare exception is here to catch spurious exceptions
# thrown by que.get during healthy operation. Please do not
# worry about this bare exception, it is entirely here to
# control program flow.
pass
for host in running:
if not running[host]['thread'].is_alive():
if host not in returned:
# Try to get any returns that came through since we
# last checked
try:
while True:
ret = que.get(False)
if 'id' in ret:
returned.add(ret['id'])
yield {ret['id']: ret['ret']}
except Exception:
pass
if host not in returned:
error = ('Target \'{0}\' did not return any data, '
'probably due to an error.').format(host)
ret = {'id': host,
'ret': error}
log.error(error)
yield {ret['id']: ret['ret']}
running[host]['thread'].join()
rets.add(host)
for host in rets:
if host in running:
running.pop(host)
if len(rets) >= len(self.targets):
break
# Sleep when limit or all threads started
if len(running) >= self.opts.get('ssh_max_procs', 25) or len(self.targets) >= len(running):
time.sleep(0.1)
def handle_ssh(self, mine=False):
'''
Spin up the needed threads or processes and execute the subsequent
routines
'''
que = multiprocessing.Queue()
running = {}
target_iter = self.targets.__iter__()
returned = set()
rets = set()
init = False
while True:
if not self.targets:
log.error('No matching targets found in roster.')
break
if len(running) < self.opts.get('ssh_max_procs', 25) and not init:
try:
host = next(target_iter)
except StopIteration:
init = True
continue
for default in self.defaults:
if default not in self.targets[host]:
self.targets[host][default] = self.defaults[default]
args = (
que,
self.opts,
host,
self.targets[host],
mine,
)
routine = MultiprocessingProcess(
target=self.handle_routine,
args=args)
routine.start()
running[host] = {'thread': routine}
continue
ret = {}
try:
ret = que.get(False)
if 'id' in ret:
returned.add(ret['id'])
yield {ret['id']: ret['ret']}
except Exception:
# This bare exception is here to catch spurious exceptions
# thrown by que.get during healthy operation. Please do not
# worry about this bare exception, it is entirely here to
# control program flow.
pass
for host in running:
if not running[host]['thread'].is_alive():
if host not in returned:
# Try to get any returns that came through since we
# last checked
try:
while True:
ret = que.get(False)
if 'id' in ret:
returned.add(ret['id'])
yield {ret['id']: ret['ret']}
except Exception:
pass
if host not in returned:
error = ('Target \'{0}\' did not return any data, '
'probably due to an error.').format(host)
ret = {'id': host,
'ret': error}
log.error(error)
yield {ret['id']: ret['ret']}
running[host]['thread'].join()
rets.add(host)
for host in rets:
if host in running:
running.pop(host)
if len(rets) >= len(self.targets):
break
# Sleep when limit or all threads started
if len(running) >= self.opts.get('ssh_max_procs', 25) or len(self.targets) >= len(running):
time.sleep(0.1)
class RAETCaller(BaseCaller):
'''
Object to wrap the calling of local salt modules for the salt-call command
when transport is raet
There are two operation modes.
1) Use a preexisting minion
2) Set up a special caller minion if no preexisting minion
The special caller minion is a subset whose only function is to perform
Salt-calls with raet as the transport
The essentials:
A RoadStack whose local estate name is of the form "role_kind" where:
role is the minion id opts['id']
kind is opts['__role'] which should be 'caller' APPL_KIND_NAMES
The RoadStack if for communication to/from a master
A LaneStack with manor yard so that RaetChannels created by the func Jobbers
can communicate through this manor yard then through the
RoadStack to/from a master
A Router to route between the stacks (Road and Lane)
These are all managed via a FloScript named caller.flo
'''
def __init__(self, opts):
'''
Pass in the command line options
'''
self.process = None
if not opts['local']:
self.stack = self._setup_caller_stack(opts)
salt.transport.jobber_stack = self.stack
if (opts.get('__role') ==
kinds.APPL_KIND_NAMES[kinds.applKinds.caller]):
# spin up and fork minion here
self.process = MultiprocessingProcess(target=raet_minion_run,
kwargs={'cleanup_protecteds': [self.stack.ha], })
self.process.start()
# wait here until '/var/run/salt/minion/alpha_caller.manor.uxd' exists
self._wait_caller(opts)
super(RAETCaller, self).__init__(opts)
def run(self):
'''
Execute the salt call logic
'''
try:
ret = self.call()
if not self.opts['local']:
self.stack.server.close()
salt.transport.jobber_stack = None
if self.opts['metadata']:
print_ret = ret
else:
print_ret = ret.get('return', {})
if self.process:
self.process.terminate()
salt.output.display_output(
{'local': print_ret},
ret.get('out', 'nested'),
self.opts)
if self.opts.get('retcode_passthrough', False):
sys.exit(ret['retcode'])
except SaltInvocationError as err:
raise SystemExit(err)
def _setup_caller_stack(self, opts):
'''
Setup and return the LaneStack and Yard used by by channel when global
not already setup such as in salt-call to communicate to-from the minion
'''
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup RAETChannel.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAETChannel.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAETChannel.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = opts['sock_dir']
stackname = 'caller' + nacling.uuid(size=18)
stack = LaneStack(name=stackname,
lanename=lanename,
sockdirpath=sockdirpath)
stack.Pk = raeting.PackKind.pack.value
stack.addRemote(RemoteYard(stack=stack,
name='manor',
lanename=lanename,
dirpath=sockdirpath))
log.debug("Created Caller Jobber Stack {0}\n".format(stack.name))
return stack
def _wait_caller(self, opts):
'''
Returns when RAET Minion Yard is available
'''
yardname = 'manor'
dirpath = opts['sock_dir']
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup RAET SaltCaller.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAET SaltCaller.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAET SaltCaller.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
ha, dirpath = Yard.computeHa(dirpath, lanename, yardname)
if is_windows():
# RAET lanes do not use files on Windows. Need to use win32file
# API to check for existence.
exists = False
while not exists:
try:
f = win32file.CreateFile(
ha,
win32file.GENERIC_WRITE | win32file.GENERIC_READ,
win32file.FILE_SHARE_READ,
None,
win32file.OPEN_EXISTING,
0,
None)
win32file.CloseHandle(f)
exists = True
except win32file.error:
time.sleep(0.1)
else:
while not ((os.path.exists(ha) and
not os.path.isfile(ha) and
not os.path.isdir(ha))):
time.sleep(0.1)
time.sleep(0.5)