Esempio n. 1
0
def _run_on_all_nodes(runner, output_dir, cmd_args, print_stderr=True):
    """Given an :py:class:`EMRJobRunner`, run the command specified by
    *cmd_args* on all nodes in the cluster and save the stdout and stderr of
    each run to subdirectories of *output_dir*.

    You should probably have run :py:meth:`_enable_slave_ssh_access()` on the
    runner before calling this function.
    """
    master_addr = runner._address_of_master()
    addresses = [master_addr]

    ssh_bin = runner._opts['ssh_bin']
    ec2_key_pair_file = runner._opts['ec2_key_pair_file']

    keyfile = None
    slave_addrs = runner.fs.ssh_slave_hosts(master_addr)

    if slave_addrs:
        addresses += [
            '%s!%s' % (master_addr, slave_addr) for slave_addr in slave_addrs
        ]
        # copying key file like a boss (name of keyfile doesn't really matter)
        keyfile = 'mrboss-%s.pem' % random_identifier()
        _ssh_copy_key(ssh_bin, master_addr, ec2_key_pair_file, keyfile)

    for addr in addresses:

        stdout, stderr = _ssh_run_with_recursion(
            ssh_bin,
            addr,
            ec2_key_pair_file,
            keyfile,
            cmd_args,
        )

        if print_stderr:
            print('---')
            print('Command completed on %s.' % addr)
            print(to_string(stderr), end=' ')

        if '!' in addr:
            base_dir = os.path.join(output_dir, 'slave ' + addr.split('!')[1])
        else:
            base_dir = os.path.join(output_dir, 'master')

        if not os.path.exists(base_dir):
            os.makedirs(base_dir)

        with open(os.path.join(base_dir, 'stdout'), 'wb') as f:
            f.write(stdout)

        with open(os.path.join(base_dir, 'stderr'), 'wb') as f:
            f.write(stderr)
Esempio n. 2
0
def _run_on_all_nodes(runner, output_dir, cmd_args, print_stderr=True):
    """Given an :py:class:`EMRJobRunner`, run the command specified by
    *cmd_args* on all nodes in the cluster and save the stdout and stderr of
    each run to subdirectories of *output_dir*.

    You should probably have run :py:meth:`_enable_slave_ssh_access()` on the
    runner before calling this function.
    """
    master_addr = runner._address_of_master()
    addresses = [master_addr]

    ssh_bin = runner._opts['ssh_bin']
    ec2_key_pair_file = runner._opts['ec2_key_pair_file']

    keyfile = None
    slave_addrs = runner.fs.ssh_slave_hosts(master_addr)

    if slave_addrs:
        addresses += ['%s!%s' % (master_addr, slave_addr)
                      for slave_addr in slave_addrs]
        # copying key file like a boss (name of keyfile doesn't really matter)
        keyfile = 'mrboss-%s.pem' % random_identifier()
        _ssh_copy_key(ssh_bin, master_addr, ec2_key_pair_file, keyfile)

    for addr in addresses:

        stdout, stderr = _ssh_run_with_recursion(
            ssh_bin,
            addr,
            ec2_key_pair_file,
            keyfile,
            cmd_args,
        )

        if print_stderr:
            print('---')
            print('Command completed on %s.' % addr)
            print(to_string(stderr), end=' ')

        if '!' in addr:
            base_dir = os.path.join(output_dir, 'slave ' + addr.split('!')[1])
        else:
            base_dir = os.path.join(output_dir, 'master')

        if not os.path.exists(base_dir):
            os.makedirs(base_dir)

        with open(os.path.join(base_dir, 'stdout'), 'wb') as f:
            f.write(stdout)

        with open(os.path.join(base_dir, 'stderr'), 'wb') as f:
            f.write(stderr)
Esempio n. 3
0
    def _key_filename_for(self, addr):
        """If *addr* is a !-separated pair of hosts like ``master!slave``,
        get the name of the copy of our keypair file on ``master``. If there
        isn't one, pick a random name, and copy the key file there.

        Otherwise, return ``None``."""
        # don't need to copy a key if we're SSHing directly
        if '!' not in addr:
            return None

        host = addr.split('!')[0]

        if host not in self._host_to_key_filename:
            # copy the key if we haven't already
            keyfile = 'mrjob-%s.pem' % random_identifier()
            _ssh_copy_key(self._ssh_bin, host, self._ec2_key_pair_file, keyfile)
            # don't set above; _ssh_copy_key() may throw an IOError
            self._host_to_key_filename[host] = keyfile

        return self._host_to_key_filename[host]
Esempio n. 4
0
    def _key_filename_for(self, addr):
        """If *addr* is a !-separated pair of hosts like ``master!slave``,
        get the name of the copy of our keypair file on ``master``. If there
        isn't one, pick a random name, and copy the key file there.

        Otherwise, return ``None``."""
        # don't need to copy a key if we're SSHing directly
        if '!' not in addr:
            return None

        host = addr.split('!')[0]

        if host not in self._host_to_key_filename:
            # copy the key if we haven't already
            keyfile = 'mrjob-%s.pem' % random_identifier()
            _ssh_copy_key(self._ssh_bin, host, self._ec2_key_pair_file,
                          keyfile)
            # don't set above; _ssh_copy_key() may throw an IOError
            self._host_to_key_filename[host] = keyfile

        return self._host_to_key_filename[host]