def setupSSHAgent():
ssh_agent = popen2.Popen4("ssh-agent") # FIXME: should be render configurable
ssh_agent_auth_sock_re = "^SSH_AUTH_SOCK=([^;]+);";
ssh_agent_pid_re = "^SSH_AGENT_PID=([^;]+);";
if not ssh_agent.wait() == 0: # exit code != 0: tell it
logging.getLogger().warn("launcher %s: Couldn't run the ssh-agent binary, hope you won't need to forward ssh keys !" % (LauncherConfig().name))
return
for line in ssh_agent.fromchild.readlines():
if re.match(ssh_agent_auth_sock_re, line):
LauncherConfig().ssh_agent_sock = re.search(ssh_agent_auth_sock_re, line).group(1)
if re.match(ssh_agent_pid_re, line):
LauncherConfig().ssh_agent_pid = int(re.search(ssh_agent_pid_re, line).group(1))
if not LauncherConfig().ssh_agent_sock:
logging.getLogger().info("launcher %s: Successfully run the ssh-agent binary but can't find auth socket" % (LauncherConfig().name))
return
os.putenv('SSH_AUTH_SOCK', LauncherConfig().ssh_agent_sock)
logging.getLogger().info("launcher %s: Successfully run the ssh-agent binary, auth socket is %s" % (LauncherConfig().name, LauncherConfig().ssh_agent_sock))
for keyname in LauncherConfig().ssh_keys:
if addPrivKeyToSSHAgent(keyname):
logging.getLogger().info("launcher %s: Successfuly declared the ssh key '%s' to your ssh agent" % (LauncherConfig().name, keyname))
else:
logging.getLogger().warn("launcher %s: Couldn't declare the ssh key '%s' to your ssh agent, hope you won't need to forward it !" % (LauncherConfig().name, keyname))
def getProxy(url):
"""
Return a suitable LauncherProxy object to communicate with scheduler
"""
verifypeer = LauncherConfig().launchers[
LauncherConfig().name]['verifypeer']
return Pulse2XMLRPCProxy(url, verifypeer=verifypeer)
def from_remote_to_launcher(command_id, client, paths, targetpath, bwlimit,
wrapper_timeout):
"""
Recursive copy of a directory from a client to the launcher using scp.
"""
client = pulse2.launcher.utils.setDefaultClientOptions(client)
if not LauncherConfig().is_scp_available:
logging.getLogger().warn(
"Can't do remote to launcher because scp is not available")
return False
real_command = [LauncherConfig().scp_path]
real_command += client['transp_args']
if bwlimit:
real_command += ['-l'] + [str(bwlimit)]
real_command += ['-r']
for path in paths:
real_command += ["%s@%s:%s" % (client['user'], client['host'], path)]
real_command += [targetpath]
# The following ssh options are not used by scp, so we remove them
real_command.remove('-T')
real_command.remove('-a')
# Built "thru" command
thru_command_list = [LauncherConfig().ssh_path]
thru_command_list += client['transp_args']
thru_command_list += ["%s@%s" % (client['user'], client['host'])]
# Build final command line
command_list = [
LauncherConfig().wrapper_path, '--max-log-size',
str(LauncherConfig().wrapper_max_log_size), '--max-exec-time',
str(wrapper_timeout), '--exec',
PULSE2_WRAPPER_ARG_SEPARATOR.join(real_command), '--thru',
PULSE2_WRAPPER_ARG_SEPARATOR.join(thru_command_list),
'--exec-server-side'
]
# from {'a': 'b', 'c: 'd'} to 'a=b,c=d'
if client['client_check']:
command_list += [
'--check-client-side', ','.join(
map((lambda x: '='.join(x)), client['client_check'].items()))
]
if client['server_check']:
command_list += [
'--check-server-side', ','.join(
map((lambda x: '='.join(x)), client['server_check'].items()))
]
if client['action']:
command_list += ['--action', client['action']]
result = pulse2.launcher.process_control.commandRunner(
command_list, __cb_sync_process_end)
if not result:
logging.getLogger().warn("Remote to launcher failed for CoH #%d" %
command_id)
return result
def __cb_icmp_end(shprocess, client=client):
if not shprocess.exit_code == 0:
logging.getLogger().debug(
"launcher %s: ICMP failed on %s: %s, %s" %
(LauncherConfig().name, client, shprocess.stdout,
shprocess.stderr))
return False
logging.getLogger().debug("launcher %s: ICMP succeeded on %s" %
(LauncherConfig().name, client))
return True
def __cb_wol_end(shprocess):
if not shprocess.exit_code == 0:
logging.getLogger().warn(
"launcher %s: WOL failed: %s, %s" %
(LauncherConfig().name, shprocess.stdout, shprocess.stderr))
return (False, "mac addresses: %s, target broadcasts: %s" %
(mac_addrs, target_bcast), "")
logging.getLogger().debug("launcher %s: WOL succeeded" %
(LauncherConfig().name))
return (True, "mac addresses: %s, target broadcasts: %s" %
(mac_addrs, target_bcast), "")
def removePrivKeyFromSSHAgent(key_name):
"""
ask the ssh-agent to keep our key
"""
if key_name == None or key_name == '':
key_name = LauncherConfig().ssh_defaultkey
if key_name not in LauncherConfig().ssh_keys.keys():
return False
return (os.system('ssh-add -d %s 2> /dev/null' % LauncherConfig().ssh_keys[key_name]) == 0)
def sendSignal(self, signal):
# signal is posix signal ID, see kill -l
logging.getLogger().debug('launcher %s: sent signal %s to command %s' %
(LauncherConfig().name, signal, self.id))
try:
self.handler.signalProcess(signal)
return True
except:
logging.getLogger().warn(
'launcher %s: sent signal %s to command %s which is already finished'
% (LauncherConfig().name, signal, self.id))
return False
def parse_result():
if LauncherConfig().tcp_sproxy_host:
tcp_sproxy_host = LauncherConfig().tcp_sproxy_host
else:
# Take the first network interface
logging.getLogger().info('tcp_sproxy_host param was not specified in launcher config')
available_interfaces = get_all_interfaces()
if available_interfaces:
tcp_sproxy_host = available_interfaces[0]['ip']
logging.getLogger().info('Using %(name)s %(ip)s as IP address for tcp_sproxy_host' % available_interfaces[0])
else:
raise Exception('No IP found for tcp_sproxy_host')
return LauncherConfig().name, tcp_sproxy_host, proxy_port, auth_key
def killSSHAgent():
if LauncherConfig().ssh_agent_pid:
logging.getLogger().info("launcher %s: terminating the ssh-agent binary (pid is %s)" % (LauncherConfig().name, LauncherConfig().ssh_agent_pid))
try:
os.kill(LauncherConfig().ssh_agent_pid, signal.SIGTERM)
time.sleep(1) # give ssh-agent 1 second to exit
try:
os.kill(LauncherConfig().ssh_agent_pid, signal.SIGKILL)
logging.getLogger().warn("launcher %s: Had to kill the ssh-agent binary" % (LauncherConfig().name))
except OSError: # pid do not exists anymore
logging.getLogger().info("launcher %s: Successfully terminated the ssh-agent binary" % (LauncherConfig().name))
except OSError: # agent do not exists anymore
logging.getLogger().warn("launcher %s: the ssh-agent binary seems already down" % (LauncherConfig().name))
def wolClient(mac_addrs, target_bcast=None):
""" Send a BCast WOL packet to mac_addrs """
def __cb_wol_end(shprocess):
if not shprocess.exit_code == 0:
logging.getLogger().warn(
"launcher %s: WOL failed: %s, %s" %
(LauncherConfig().name, shprocess.stdout, shprocess.stderr))
return (False, "mac addresses: %s, target broadcasts: %s" %
(mac_addrs, target_bcast), "")
logging.getLogger().debug("launcher %s: WOL succeeded" %
(LauncherConfig().name))
return (True, "mac addresses: %s, target broadcasts: %s" %
(mac_addrs, target_bcast), "")
def cbReturn(result):
ret = (True, "mac addresses: %s, target broadcasts: %s" %
(mac_addrs, target_bcast), "")
for res in result:
if not res[1]:
return (False, "mac addresses: %s, target broadcasts: %s" %
(mac_addrs, target_bcast), "")
return ret
command_list = [
LauncherConfig().wol_path,
'--port=%s' % LauncherConfig().wol_port,
]
dl = []
sorted = {}
for i in range(len(mac_addrs)):
if mac_addrs[i]:
bcast = LauncherConfig().wol_bcast
if target_bcast[i]:
bcast = target_bcast[i]
if not sorted.has_key(bcast):
sorted[bcast] = []
sorted[bcast].append(mac_addrs[i])
for bcat in sorted:
mac_addresses = sorted[bcat]
cmd = command_list + ['--ipaddr=%s' % bcat] + mac_addresses
logging.getLogger().debug("launcher %s: WOL: %s" %
(str(cmd), LauncherConfig().name))
dl.append(
pulse2.launcher.process_control.commandRunner(cmd, __cb_wol_end))
if len(dl) == 1:
return dl[0]
dl = defer.DeferredList(dl)
dl.addCallback(cbReturn)
return dl
def __cb_probe_end(result, client=client):
(exitcode, stdout, stderr) = result
idData = [{
'platform': "Microsoft Windows",
'pcre': "Windows",
"tmp_path": "/lsc",
"root_path": "/cygdrive/c"
}, {
'platform': "GNU Linux",
'pcre': "Linux",
"tmp_path": "/tmp/lsc",
"root_path": "/"
}, {
'platform': "Sun Solaris",
'pcre': "SunOS",
"tmp_path": "/tmp/lsc",
"root_path": "/"
}, {
'platform': "IBM AIX",
'pcre': "AIX",
"tmp_path": "/tmp/lsc",
"root_path": "/"
}, {
'platform': "HP UX",
'pcre': "HP-UX",
"tmp_path": "/tmp/lsc",
"root_path": "/"
}, {
'platform': "Apple MacOS",
'pcre': "Darwin",
"tmp_path": "/tmp/lsc",
"root_path": "/"
}]
if not exitcode == 0:
logging.getLogger().debug(
"launcher %s: PROBE execution failed on %s: %s, %s" %
(LauncherConfig().name, client, stdout, stderr))
return "Not available"
for identification in idData:
if re.compile(identification["pcre"]).search(
stdout) or stdout == identification["platform"]:
logging.getLogger().debug(
"launcher %s: PROBE identification succeded on %s: %s" %
(LauncherConfig().name, client,
identification["platform"]))
return identification["platform"]
logging.getLogger().debug(
"launcher %s: PROBE identification failed on %s: %s, %s" %
(LauncherConfig().name, client, stdout, stderr))
return "Other"
def __cb_async_process_end(shprocess):
"""
Handle async process termination
"""
def _cb(result, id):
# as we successfuly sent our result to our scheduler, command can be safely removed from our list
shprocess.isnotifyingparent = False
if result == True:
pulse2.launcher.process_control.ProcessList().removeProcess(id)
else:
logging.getLogger().warn('launcher %s: failed to send results of command #%s to our scheduler at %s: %s' % (LauncherConfig().name, id, scheduler, "command was rejected; lock issue?"))
def _eb(reason, id):
# no result can be sent, log and keep our process in our list
shprocess.isnotifyingparent = False
logging.getLogger().warn('launcher %s: failed to send results of command #%s to our scheduler at %s: %s' % (LauncherConfig().name, id, scheduler, reason.value))
if not shprocess.isnotifyingparent:
shprocess.isnotifyingparent = True
exitcode = shprocess.exit_code
stdout = unicode(shprocess.stdout, 'utf-8', 'strict')
stderr = unicode(shprocess.stderr, 'utf-8', 'strict')
id = shprocess.id
scheduler = pulse2.launcher.utils.getScheduler()
mydeffered = getProxy(scheduler).callRemote(
shprocess.returnxmlrpcfunc,
LauncherConfig().name,
(exitcode, stdout, stderr),
id
)
mydeffered.\
addCallback(_cb, id).\
addErrback(_eb, id)
return
def commandRunner(cmd, cbCommandEnd):
"""
Return a Deferred resulting in the stdout output of a shell command.
Only used in sync mode.
"""
process = commandProtocol(cmd)
# FIXME: codec should be taken from conf file
try:
process.handler = twisted.internet.reactor.spawnProcess(
process,
cmd[0],
map(lambda (x): x.encode('utf-8', 'ignore'), cmd),
None, # env
None, # path
None, # uid
None, # gid
None, # usePTY
{
0: "w",
1: 'r',
2: 'r'
} # FDs: not closing STDIN (might be used)
)
except OSError, e:
logging.getLogger().error(
'launcher %s: failed daemonization in commandRunner: %d (%s)' %
(LauncherConfig().name, e.errno, e.strerror))
return False
def commandForker(cmd,
cbCommandEnd,
id,
defer_results,
callbackName,
max_exec_time,
group,
kind,
env_={}):
"""
"""
if ProcessList().existsProcess(id):
logging.getLogger().warn(
'launcher %s: attempted to add command #%s twice' %
(LauncherConfig().name, id))
return False
if not ProcessList().isOneSlotFree():
logging.getLogger().warn(
'launcher %s: running out of slot when adding command #%s' %
(LauncherConfig().name, id))
return False
process = commandProtocol(cmd)
process.id = id
env_ = _checkEnvArgs(env_=env_)
# FIXME: codec should be taken from conf file
try:
process.handler = twisted.internet.reactor.spawnProcess(
process,
cmd[0],
map(lambda (x): x.encode('utf-8', 'ignore'), cmd),
env=env_,
childFDs={
1: 'r',
2: 'r'
})
except OSError, e:
logging.getLogger().error(
'launcher %s: failed daemonization in commandForker: %d (%s)' %
(LauncherConfig().name, e.errno, e.strerror))
# do some cleanup
return False
def allocate_port_couple():
"""
Looking for two free ports to establish SSH proxy.
@return: two free ports
@rtype: list
"""
ret_ports = []
port_range = range(LauncherConfig().tcp_sproxy_port_range_start + 1,
LauncherConfig().tcp_sproxy_port_range_end + 1)
for port in port_range :
if NetUtils.is_port_free(port):
ret_ports.append(port)
if len(ret_ports) == 2 :
logging.getLogger().debug("Allocated ports to build TCP SSH Proxy: (%d:%d)" % tuple(ret_ports))
return ret_ports
def getPubKey(key_name):
"""
Handle remote download of this launcher's pubkey.
key_name is as define in the config file
"""
try:
LauncherConfig().ssh_keys[key_name]
except KeyError:
key_name = LauncherConfig().ssh_defaultkey
if key_name == None or key_name == '':
key_name = LauncherConfig().ssh_defaultkey
try:
ssh_key = open(LauncherConfig().ssh_keys[key_name] + '.pub')
except IOError: # key does not exists, give up
return ''
ret = ' '.join(ssh_key)
ssh_key.close()
return ret
def _cb(result, id):
# as we successfuly sent our result to our scheduler, command can be safely removed from our list
shprocess.isnotifyingparent = False
if result == True:
pulse2.launcher.process_control.ProcessList().removeProcess(id)
else:
logging.getLogger().warn(
'launcher %s: failed to send results of command #%s to our scheduler at %s: %s'
% (LauncherConfig().name, id, scheduler,
"command was rejected; lock issue?"))
def get_command_result(command_id, command_list, mode, group, envir, step,
step_log):
if mode == 'async':
result = pulse2.launcher.process_control.commandForker(
command_list,
__cb_async_process_end,
command_id,
LauncherConfig().defer_results,
'completed_' + step,
LauncherConfig().max_command_age,
group,
step,
env_=envir)
elif mode == 'sync':
result = pulse2.launcher.process_control.commandRunner(
command_list, __cb_sync_process_end, env_=envir)
if not result:
logging.getLogger().warn("%s failed for CoH #%d" %
(step_log, command_id))
return result
def killOldCommands(self):
""" attempt to kill out-of-time commands """
if self.getRunningCount() > 0:
for id in self.getRunningIds():
process = self.getProcess(id)
times = process.getTimes()
# priority check order: use process.max_age if not 0, else use self.default_timeout if not 0
if not process.max_age == 0:
if times['age'] > process.max_age: # kill time
logging.getLogger().warn(
'launcher %s: killing %s (out of time: current %s, max %s)'
% (LauncherConfig().name, id, times['age'],
process.max_age))
killProcess(id)
elif not self.default_timeout == 0:
if times['age'] > self.default_timeout: # kill time
logging.getLogger().warn(
'launcher %s: killing %s (out of time: current %s, max %s)'
% (LauncherConfig().name, id, times['age'],
self.default_timeout))
killProcess(id)
def getScheduler():
""" Get our referent scheduler """
config = LauncherConfig()
if config.scheduler_enablessl:
uri = 'https://'
else:
uri = 'http://'
if config.scheduler_username != '':
uri += '%s:%s@' % (config.scheduler_username,
config.scheduler_password)
uri += '%s:%s' % (config.scheduler_host, config.scheduler_port)
return uri
def icmpClient(client, timeout):
""" Send a Ping to our client """
def __cb_icmp_end(shprocess, client=client):
if not shprocess.exit_code == 0:
logging.getLogger().debug(
"launcher %s: ICMP failed on %s: %s, %s" %
(LauncherConfig().name, client, shprocess.stdout,
shprocess.stderr))
return False
logging.getLogger().debug("launcher %s: ICMP succeeded on %s" %
(LauncherConfig().name, client))
return True
command_list = [LauncherConfig().ping_path, client]
# FIXME: use timeout
return pulse2.launcher.process_control.commandRunner(
command_list,
__cb_icmp_end,
)
def commandRunner(cmd, cbCommandEnd, env_={}):
"""
Return a Deferred resulting in the stdout output of a shell command.
Only used in sync mode.
"""
process = commandProtocol(cmd)
env_ = _checkEnvArgs(env_=env_)
# FIXME: codec should be taken from conf file
try:
process.handler = twisted.internet.reactor.spawnProcess(
process,
cmd[0],
map(lambda (x): x.encode('utf-8', 'ignore'), cmd),
env=env_,
childFDs={
1: 'r',
2: 'r'
})
except OSError, e:
logging.getLogger().error(
'launcher %s: failed daemonization in commandRunner: %d (%s)' %
(LauncherConfig().name, e.errno, e.strerror))
return False
def remote_halt(command_id, client, mode, wrapper_timeout):
""" Handle remote halt on target
"""
client = pulse2.launcher.utils.setDefaultClientOptions(client)
halt_command = LauncherConfig().halt_command
if client['protocol'] == "ssh":
# command is issued though our wrapper, time to build it
if not LauncherConfig().is_ssh_available:
logging.getLogger().warn(
"Can't do remote halt because ssh is not available")
return False
# Built "thru" command
thru_command_list = [LauncherConfig().ssh_path]
thru_command_list += client['transp_args']
thru_command_list += [client['host']]
# Define environnement
envir = {
'SSHPATH': LauncherConfig().ssh_path,
'SSHARGS': client['transp_args'],
'SSHTARGET': client['host'],
'STEP': 'HALT'
}
# Build "exec" command
real_command = halt_command
# Build final command line
command_list = [
LauncherConfig().wrapper_path,
'--max-log-size',
str(LauncherConfig().wrapper_max_log_size),
'--max-exec-time',
str(wrapper_timeout),
'--thru',
PULSE2_WRAPPER_ARG_SEPARATOR.join(thru_command_list),
'--exec',
real_command, # we do not use the PULSE2_WRAPPER_ARG_SEPARATOR here, as the command is send "as is"
]
# from {'a': 'b', 'c: 'd'} to 'a=b,c=d'
if client['client_check']:
command_list += [
'--check-client-side', ','.join(
map((lambda x: '='.join(x)),
client['client_check'].items()))
]
if client['server_check']:
command_list += [
'--check-server-side', ','.join(
map((lambda x: '='.join(x)),
client['server_check'].items()))
]
if client['action']:
command_list += ['--action', client['action']]
return get_command_result(command_id, command_list, mode,
client['group'], envir, 'halt',
'Remote halt (ssh/async)')
logging.getLogger().warn("Remote halt failed for CoH #%d" % command_id)
return None
process,
cmd[0],
map(lambda (x): x.encode('utf-8', 'ignore'), cmd),
env=env_,
childFDs={
1: 'r',
2: 'r'
})
except OSError, e:
logging.getLogger().error(
'launcher %s: failed daemonization in commandRunner: %d (%s)' %
(LauncherConfig().name, e.errno, e.strerror))
return False
logging.getLogger().debug(
'launcher %s: about to execute %s in commandRunner' %
(LauncherConfig().name, ' '.join(cmd)))
process.deferred = twisted.internet.defer.Deferred()
process.deferred.addCallback(cbCommandEnd)
return process.deferred
@HasSufficientMemory(80)
def commandForker(cmd,
cbCommandEnd,
id,
defer_results,
callbackName,
max_exec_time,
group,
kind,
env_={}):
def _eb(reason, id):
# no result can be sent, log and keep our process in our list
shprocess.isnotifyingparent = False
logging.getLogger().warn(
'launcher %s: failed to send results of command #%s to our scheduler at %s: %s'
% (LauncherConfig().name, id, scheduler, reason.value))
def establishProxy(client, requestor_ip, requested_port):
"""
Establish a TCP connection to client using our proxy
"""
client = pulse2.launcher.utils.setDefaultClientOptions(client)
"""
client['client_check'] = getClientCheck(client)
client['server_check'] = getServerCheck(client)
client['action'] = getAnnounceCheck('vnc')
"""
def generate_auth_key():
import random
chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789$-_.+!*()'
size = 15
return ''.join(random.choice(chars) for x in range(size))
if LauncherConfig().create_web_proxy:
auth_key = generate_auth_key()
else:
auth_key = '-'
proxy_port, local_port = allocate_port_couple()
# Built "exec" command
real_command = [
LauncherConfig().tcp_sproxy_path,
requestor_ip,
client['host'],
requested_port,
','.join(client['transp_args']),
str(proxy_port),
str(local_port),
str(LauncherConfig().tcp_sproxy_establish_delay),
str(LauncherConfig().tcp_sproxy_connect_delay),
str(LauncherConfig().tcp_sproxy_session_lenght),
client['shortname'],
auth_key,
]
# Built "thru" command
thru_command_list = [LauncherConfig().ssh_path]
for option in client['transp_args']:
thru_command_list += ['-o', option]
thru_command_list += [client['host']]
command_list = [
LauncherConfig().wrapper_path,
'--max-log-size',
str(LauncherConfig().wrapper_max_log_size),
#'--max-exec-time', # FIXME: wrapper_timeout missing in function signature :/
#str(wrapper_timeout),
'--exec',
SEPARATOR.join(real_command),
'--thru',
SEPARATOR.join(thru_command_list),
'--no-wrap',
'--only-stdout',
'--remove-empty-lines',
'--exec-server-side'
]
# from {'a': 'b', 'c: 'd'} to 'a=b,c=d'
if client['client_check']:
command_list += ['--check-client-side', ','.join(map((lambda x: '='.join(x)), client['client_check'].items()))]
if client['server_check']:
command_list += ['--check-server-side', ','.join(map((lambda x: '='.join(x)), client['server_check'].items()))]
if client['action']:
command_list += ['--action', client['action']]
proxy = proxyProtocol()
twisted.internet.reactor.spawnProcess(
proxy,
command_list[0],
map(lambda(x): x.encode('utf-8', 'ignore'), command_list),
None, # env
None, # path
None, # uid
None, # gid
None, # usePTY
{ 0: "w", 1: 'r', 2: 'r' } # FDs: not closing STDIN (might be used)
)
def get_ip_address(ifname):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
return socket.inet_ntoa(fcntl.ioctl(
s.fileno(),
0x8915, # SIOCGIFADDR
struct.pack('256s', ifname[:15])
)[20:24])
def get_all_interfaces():
"""
Return list of all available interfaces with IP
@see: https://gist.github.com/pklaus/289646
"""
def format_ip(addr):
return '.'.join([str(ord(x)) for x in addr])
max_possible = 128 # arbitrary. raise if needed.
bytes = max_possible * 32
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
names = array.array('B', '\0' * bytes)
outbytes = struct.unpack('iL', fcntl.ioctl(
s.fileno(),
0x8912, # SIOCGIFCONF
struct.pack('iL', bytes, names.buffer_info()[0])
))[0]
namestr = names.tostring()
lst = []
for i in range(0, outbytes, 40):
name = namestr[i:i+16].split('\0', 1)[0]
ip = format_ip(namestr[i+20:i+24])
if not ip.startswith('127.'):
lst.append({
'name': name,
'ip': ip,
})
return lst
def parse_result():
if LauncherConfig().tcp_sproxy_host:
tcp_sproxy_host = LauncherConfig().tcp_sproxy_host
else:
# Take the first network interface
logging.getLogger().info('tcp_sproxy_host param was not specified in launcher config')
available_interfaces = get_all_interfaces()
if available_interfaces:
tcp_sproxy_host = available_interfaces[0]['ip']
logging.getLogger().info('Using %(name)s %(ip)s as IP address for tcp_sproxy_host' % available_interfaces[0])
else:
raise Exception('No IP found for tcp_sproxy_host')
return LauncherConfig().name, tcp_sproxy_host, proxy_port, auth_key
# Waiting to establish the proxy
ret = task.deferLater(twisted.internet.reactor, 2, parse_result)
logging.getLogger().debug('about to execute ' + ' '.join(command_list))
return ret
def remote_push(command_id, client, files_list, mode, wrapper_timeout):
""" Handle remote copy (push) """
target_path = os.path.join(
LauncherConfig().target_path,
pulse2.launcher.utils.getTempFolderName(command_id, client['uuid']))
client = pulse2.launcher.utils.setDefaultClientOptions(client)
if client['protocol'] == "rsyncssh":
# command is issued though our wrapper, time to build it
real_files_list = files_list
if not LauncherConfig().is_rsync_available:
logging.getLogger().warn(
'Can\'t do remote push because rsync is not available')
return False
# Build "exec" command
real_command = [LauncherConfig().rsync_path]
real_command += client['proto_args']
real_command += real_files_list
real_command += [
'%s@%s:%s/' % (client['user'], client['host'], target_path)
]
# Build "thru" command
thru_command_list = [LauncherConfig().ssh_path]
thru_command_list += client['transp_args']
thru_command_list += ["%s@%s" % (client['user'], client['host'])]
# Define environnement
envir = {
'SSHPATH': LauncherConfig().ssh_path,
'SSHARGS': client['transp_args'],
'SSHTARGET': "%s@%s" % (client['user'], client['host']),
'STEP': 'PUSH'
}
# Build final command line
command_list = [
LauncherConfig().wrapper_path, '--max-log-size',
str(LauncherConfig().wrapper_max_log_size), '--max-exec-time',
str(wrapper_timeout), '--exec',
PULSE2_WRAPPER_ARG_SEPARATOR.join(real_command), '--thru',
PULSE2_WRAPPER_ARG_SEPARATOR.join(thru_command_list),
'--exec-server-side'
]
# from {'a': 'b', 'c: 'd'} to 'a=b,c=d'
if client['client_check']:
command_list += [
'--check-client-side', ','.join(
map((lambda x: '='.join(x)),
client['client_check'].items()))
]
if client['server_check']:
command_list += [
'--check-server-side', ','.join(
map((lambda x: '='.join(x)),
client['server_check'].items()))
]
if client['action']:
command_list += ['--action', client['action']]
return get_command_result(command_id, command_list, mode,
client['group'], envir, 'push',
'Remote push (rsyncssh/async)')
logging.getLogger().warn("Remote push failed for CoH #%d" % command_id)
return None
def remote_delete(command_id, client, files_list, mode, wrapper_timeout):
""" Handle remote deletion on target """
client = pulse2.launcher.utils.setDefaultClientOptions(client)
target_path = os.path.join(
LauncherConfig().target_path,
pulse2.launcher.utils.getTempFolderName(command_id, client['uuid']))
if client['protocol'] == "ssh":
# command is issued through our wrapper, time to build it
if not LauncherConfig().is_ssh_available:
logging.getLogger().warn(
"Can't do remote delete because ssh is not available")
return False
# Built "thru" command
thru_command_list = [LauncherConfig().ssh_path]
thru_command_list += client['transp_args']
thru_command_list += [client['host']]
# Define environnement
envir = {
'SSHPATH': LauncherConfig().ssh_path,
'SSHARGS': client['transp_args'],
'SSHTARGET': client['host'],
'STEP': 'DELETE'
}
# Build "exec" command
# The permissions need to be modified, else the directory can't be
# deleted.
real_command = ['if', 'id', 'SYSTEM', '>', '/dev/null', ';', 'then']
real_command += ['chown', 'SYSTEM.SYSTEM', '"%s"' % target_path, ';']
real_command += ['fi', ';']
if LauncherConfig().is_smart_cleaner_available:
real_command += [
'if', '[', '-x',
LauncherConfig().smart_cleaner_path, ']', ';'
]
real_command += ['then']
real_command += [LauncherConfig().smart_cleaner_path]
real_command += ['--directory', '"%s"' % target_path]
real_command += ['--files', '"%s"' % ','.join(files_list)]
real_command += LauncherConfig().smart_cleaner_options
real_command += [';']
real_command += ['else']
real_command += ['rm']
real_command += map(lambda (a): '"%s"' % os.path.join(target_path, a),
files_list)
real_command += ['&&', 'if', '!', 'rmdir', target_path, ';']
real_command += ['then']
# Use the dellater command if available
real_command += ['if', '[', '-x', '/usr/bin/dellater.exe', ']', ';']
real_command += ['then']
# The mount/grep/sed stuff is needed to get the directory name for
# Windows.
real_command += [
'dellater',
'"$(mount | grep " on / type" | sed "s| on / type.*$||")"' +
target_path, ';'
]
real_command += ['fi', ';']
real_command += ['fi', ';']
if LauncherConfig().is_smart_cleaner_available:
real_command += ['fi', ';']
# Build final command line
command_list = [
LauncherConfig().wrapper_path,
'--max-log-size',
str(LauncherConfig().wrapper_max_log_size),
'--max-exec-time',
str(wrapper_timeout),
'--thru',
PULSE2_WRAPPER_ARG_SEPARATOR.join(thru_command_list),
'--exec',
PULSE2_WRAPPER_ARG_SEPARATOR.join(real_command),
]
# from {'a': 'b', 'c: 'd'} to 'a=b,c=d'
if client['client_check']:
command_list += [
'--check-client-side', ','.join(
map((lambda x: '='.join(x)),
client['client_check'].items()))
]
if client['server_check']:
command_list += [
'--check-server-side', ','.join(
map((lambda x: '='.join(x)),
client['server_check'].items()))
]
if client['action']:
command_list += ['--action', client['action']]
return get_command_result(command_id, command_list, mode,
client['group'], envir, 'delete',
'Remote delete (ssh/async)')
logging.getLogger().warn("Remote delete failed for CoH #%d" % command_id)
return None
def remote_pull(command_id, client, files_list, mode, wrapper_timeout):
""" Handle remote copy (pull) on target """
client = pulse2.launcher.utils.setDefaultClientOptions(client)
target_path = os.path.join(
LauncherConfig().target_path,
pulse2.launcher.utils.getTempFolderName(command_id, client['uuid']))
if client['protocol'] == "wget":
# command is issued though our wrapper, time to build it
if not LauncherConfig().is_ssh_available:
logging.getLogger().warn(
"Can't do remote pull because ssh is not available")
return False
# Built "thru" command
thru_command_list = [LauncherConfig().ssh_path]
thru_command_list += client['transp_args']
thru_command_list += ["%s@%s" % (client['user'], client['host'])]
# Define environnement
envir = {
'SSHPATH': LauncherConfig().ssh_path,
'SSHARGS': client['transp_args'],
'SSHTARGET': "%s@%s" % (client['user'], client['host']),
'STEP': 'PUSHPULL'
}
# Build "exec" command
real_command = [LauncherConfig().wget_path]
real_command += client['proto_args']
real_command += ['-N']
real_command += files_list
real_command += ['-P']
real_command += [target_path]
# Make downloaded files executable
real_command += ['&&']
real_command += ['chmod']
real_command += ['u+x']
real_command += ['-R']
real_command += [target_path]
# Check files integrity with MD5SUMS file if available AND size > 0
real_command += ['&&']
real_command += ['cd', target_path]
real_command += ['&&']
real_command += ['if', '[', '-s', 'MD5SUMS', ']',
';'] # '-s' : size > 0
real_command += ['then']
real_command += ['md5sum', '-c', 'MD5SUMS', ';']
real_command += ['fi']
# Build final command line
command_list = [
LauncherConfig().wrapper_path,
'--max-log-size',
str(LauncherConfig().wrapper_max_log_size),
'--max-exec-time',
str(wrapper_timeout),
'--thru',
PULSE2_WRAPPER_ARG_SEPARATOR.join(thru_command_list),
'--exec',
PULSE2_WRAPPER_ARG_SEPARATOR.join(real_command),
]
# from {'a': 'b', 'c: 'd'} to 'a=b,c=d'
if client['client_check']:
command_list += [
'--check-client-side', ','.join(
map((lambda x: '='.join(x)),
client['client_check'].items()))
]
if client['server_check']:
command_list += [
'--check-server-side', ','.join(
map((lambda x: '='.join(x)),
client['server_check'].items()))
]
if client['action']:
command_list += ['--action', client['action']]
return get_command_result(command_id, command_list, mode,
client['group'], envir, 'pull',
'Remote pull (wget/async)')
elif client['protocol'] == "rsyncproxy":
# Built "thru" command
thru_command_list = [LauncherConfig().ssh_path]
thru_command_list += client['transp_args']
thru_command_list += ["%s@%s" % (client['user'], client['host'])]
# Define environnement
envir = {
'SSHPATH': LauncherConfig().ssh_path,
'SSHARGS': client['transp_args'],
'SSHTARGET': "%s@%s" % (client['user'], client['host']),
'STEP': 'PUSHPULL'
}
# Build "exec" command
src_path = pulse2.launcher.utils.getTempFolderName(
client['proxy']['command_id'], client['proxy']['uuid'])
real_command = ['rsync']
real_command += client['proto_args']
real_command += [
'%s@%s:"%s"' % (client['user'], client['proxy']['host'], ' '.join(
map(
lambda x: "%s/%s/'%s'" %
(LauncherConfig().target_path, src_path, x), files_list)))
]
real_command += [target_path]
# Build final command line
command_list = [
LauncherConfig().wrapper_path,
'--max-log-size',
str(LauncherConfig().wrapper_max_log_size),
'--max-exec-time',
str(wrapper_timeout),
'--thru',
PULSE2_WRAPPER_ARG_SEPARATOR.join(thru_command_list),
'--exec',
PULSE2_WRAPPER_ARG_SEPARATOR.join(real_command),
]
# from {'a': 'b', 'c: 'd'} to 'a=b,c=d'
if client['client_check']:
command_list += [
'--check-client-side', ','.join(
map((lambda x: '='.join(x)),
client['client_check'].items()))
]
if client['server_check']:
command_list += [
'--check-server-side', ','.join(
map((lambda x: '='.join(x)),
client['server_check'].items()))
]
if client['action']:
command_list += ['--action', client['action']]
return get_command_result(command_id, command_list, mode,
client['group'], envir, 'pull',
'Remote pull (rsyncproxy/async)')
logging.getLogger().warn("Remote pull failed for CoH #%d" % command_id)
return None
def __err_dl_end(failure, path):
logging.getLogger().error("launcher %s: %s" %
(LauncherConfig().name, failure))
shutil.rmtree(path)
return False
