def _run_on_all_nodes(runner, output_dir, cmd_args, print_stderr=True):
"""Given an :py:class:`EMRJobRunner`, run the command specified by
*cmd_args* on all nodes in the cluster and save the stdout and stderr of
each run to subdirectories of *output_dir*.
You should probably have run :py:meth:`_enable_slave_ssh_access()` on the
runner before calling this function.
"""
master_addr = runner._address_of_master()
addresses = [master_addr]
ssh_bin = runner._opts['ssh_bin']
ec2_key_pair_file = runner._opts['ec2_key_pair_file']
keyfile = None
slave_addrs = runner.fs.ssh_slave_hosts(master_addr)
if slave_addrs:
addresses += [
'%s!%s' % (master_addr, slave_addr) for slave_addr in slave_addrs
]
# copying key file like a boss (name of keyfile doesn't really matter)
keyfile = 'mrboss-%s.pem' % random_identifier()
_ssh_copy_key(ssh_bin, master_addr, ec2_key_pair_file, keyfile)
for addr in addresses:
stdout, stderr = _ssh_run_with_recursion(
ssh_bin,
addr,
ec2_key_pair_file,
keyfile,
cmd_args,
)
if print_stderr:
print('---')
print('Command completed on %s.' % addr)
print(to_string(stderr), end=' ')
if '!' in addr:
base_dir = os.path.join(output_dir, 'slave ' + addr.split('!')[1])
else:
base_dir = os.path.join(output_dir, 'master')
if not os.path.exists(base_dir):
os.makedirs(base_dir)
with open(os.path.join(base_dir, 'stdout'), 'wb') as f:
f.write(stdout)
with open(os.path.join(base_dir, 'stderr'), 'wb') as f:
f.write(stderr)
def _run_on_all_nodes(runner, output_dir, cmd_args, print_stderr=True):
"""Given an :py:class:`EMRJobRunner`, run the command specified by
*cmd_args* on all nodes in the cluster and save the stdout and stderr of
each run to subdirectories of *output_dir*.
You should probably have run :py:meth:`_enable_slave_ssh_access()` on the
runner before calling this function.
"""
master_addr = runner._address_of_master()
addresses = [master_addr]
ssh_bin = runner._opts['ssh_bin']
ec2_key_pair_file = runner._opts['ec2_key_pair_file']
keyfile = None
slave_addrs = runner.fs.ssh_slave_hosts(master_addr)
if slave_addrs:
addresses += ['%s!%s' % (master_addr, slave_addr)
for slave_addr in slave_addrs]
# copying key file like a boss (name of keyfile doesn't really matter)
keyfile = 'mrboss-%s.pem' % random_identifier()
_ssh_copy_key(ssh_bin, master_addr, ec2_key_pair_file, keyfile)
for addr in addresses:
stdout, stderr = _ssh_run_with_recursion(
ssh_bin,
addr,
ec2_key_pair_file,
keyfile,
cmd_args,
)
if print_stderr:
print('---')
print('Command completed on %s.' % addr)
print(to_string(stderr), end=' ')
if '!' in addr:
base_dir = os.path.join(output_dir, 'slave ' + addr.split('!')[1])
else:
base_dir = os.path.join(output_dir, 'master')
if not os.path.exists(base_dir):
os.makedirs(base_dir)
with open(os.path.join(base_dir, 'stdout'), 'wb') as f:
f.write(stdout)
with open(os.path.join(base_dir, 'stderr'), 'wb') as f:
f.write(stderr)
def _key_filename_for(self, addr):
"""If *addr* is a !-separated pair of hosts like ``master!slave``,
get the name of the copy of our keypair file on ``master``. If there
isn't one, pick a random name, and copy the key file there.
Otherwise, return ``None``."""
# don't need to copy a key if we're SSHing directly
if '!' not in addr:
return None
host = addr.split('!')[0]
if host not in self._host_to_key_filename:
# copy the key if we haven't already
keyfile = 'mrjob-%s.pem' % random_identifier()
_ssh_copy_key(self._ssh_bin, host, self._ec2_key_pair_file, keyfile)
# don't set above; _ssh_copy_key() may throw an IOError
self._host_to_key_filename[host] = keyfile
return self._host_to_key_filename[host]
def _key_filename_for(self, addr):
"""If *addr* is a !-separated pair of hosts like ``master!slave``,
get the name of the copy of our keypair file on ``master``. If there
isn't one, pick a random name, and copy the key file there.
Otherwise, return ``None``."""
# don't need to copy a key if we're SSHing directly
if '!' not in addr:
return None
host = addr.split('!')[0]
if host not in self._host_to_key_filename:
# copy the key if we haven't already
keyfile = 'mrjob-%s.pem' % random_identifier()
_ssh_copy_key(self._ssh_bin, host, self._ec2_key_pair_file,
keyfile)
# don't set above; _ssh_copy_key() may throw an IOError
self._host_to_key_filename[host] = keyfile
return self._host_to_key_filename[host]