Exemple #1
0
def config():
    """
    Configure RCluster and AWS EC2 account.
    Prompts user for credentials, builds an AMI with specified R packages
    installed, and saves out the configuration file with credentials to a hidden
    folder in the user's home directory.
    """
    import shutil
    parser.add_argument('-o', '--outfile', type=str, nargs=1,
                        default=rcl._set_data('json'),
                        help='The file in which to save the RCluster' +
                             'configuration data (stored in JSON format)')
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel)

    setup_cl = rcl.RCluster.from_config(rcl._get_data('config.json'), purge=True)
    setup_cl.write_config(args.outfile)
    setup_script = shutil.copyfile(rcl._get_data('ami.sh'), rcl._set_data('sh'))
    # TODO: validate inputs
    pswd = input("Enter `cluster` user password: "******"Enter R packages to install (in format: `dplyr,plyr,etc`): ")
    pkgs = '"' + '", "'.join(pkgs.split(",")) + '"'
    with open(setup_script, 'a', newline='') as script:
        script.write('echo "cluster:{0}" | chpasswd\n'.format(pswd))
        script.write(("R --vanilla -q -e 'install.packages(c({0}), repo = "
                      "\"https://cran.cnr.berkeley.edu/\")'\n").format(pkgs))
    setup_cl.create_ami(setup_fn=setup_script)
    setup_cl.write_config(args.outfile)
Exemple #2
0
def terminate():
    """
    Terminate all AWS instances associated with the specified RCluster
    configuration file.
    """
    parser.add_argument('-c', '--config', type=str, nargs=1,
                        default=rcl._set_data('json'),
                        help='The JSON RCluster configuration file.')
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel)

    cluster = rcl.RCluster.from_config(args.config)
    cluster.terminate_instances()
Exemple #3
0
def retrieve_cluster():
    """
    Retrieve the access IP address of the current manager instance (if live).
    Also opens a browser to the manager's RStudio Server.
    """
    parser.add_argument('-c', '--config', type=str, nargs=1,
                        default=rcl._set_data('json'),
                        help='The JSON RCluster configuration file.')
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel)
    log = logging.getLogger()

    cluster = rcl.RCluster.from_config(args.config)
    ip = cluster.get_manager_ip()
    if ip:
        log.debug(ip)
        _open_ip(ip)
Exemple #4
0
def main():
    """Launch an RCluster using the information saved to a configuration file"""
    parser.add_argument('-w', '--workers', type=int, nargs=1, default=[1],
                        help='The number of workers to launch.')
    parser.add_argument('-t', '--type', type=str, nargs=1, default='m4.large',
                        help='The instance type to use.')
    parser.add_argument('-c', '--config', type=str, nargs=1,
                        default=rcl._set_data('json'),
                        help='The JSON RCluster configuration file.')
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel)
    log = logging.getLogger()

    try:
        cluster = rcl.RCluster.from_config(args.config)
    except FileNotFoundError as err:
        log.err('Run `rcluster-config`, first, to generate your own config',
                'file with the minimum necessary data to start an R cluster.')
        raise err
    ip = cluster.get_manager_ip()
    if ip:
        log.info("Active rcluster found.\n",
                 "Run `rcluster-terminate` to remove the previous cluster.\n"
                 "Returning current manager instance.\n")
    else:
        cluster.create_cluster(args.workers[0], InstanceType=args.type)
        ip = cluster.access_ip
    _open_ip(ip)
    cl_data = ('Manager IP Address:', ip)
    log.info(cl_data)
    term = ''
    while term not in 'yn':
        term = input("""
        Type 'y' to terminate the cluster and exit.
        Type 'n' to exit without terminating the cluster.
        You can always terminate your current RCluster by running
        `rcluster-terminate` from the command line.
        """)
    if term == 'y':
        cluster.terminate_instances()
Exemple #5
0
    def __init__(self, aws_access_key_id, aws_secret_access_key, region_name,
                 instance_conf, manager_runtime=None, worker_runtime=None,
                 key_path=None, ip_ref='public_ip_address', ver=rcl.__ver__,
                 purge=False):
        """Initialize the RCluster object.
        
        :param aws_access_key_id: AWS access key provided to
            boto3.session.Session()
        :param aws_secret_access_key: AWS secret access key provided to
            boto3.session.Session()
        :param region_name: The accessibility region provided to
            boto3.session.Session()
        :param instance_conf: Dictionary defining {'ami': '', 'type': ''} for
            instances (where 'ami' is the AMI ID for the instances and type is
            the instance type used); can also contain other parameters to
            boto3's EC2.ServiceResource.create_instances
        :param manager_runtime: String containing shell runtime command for the
            manager instance
        :param worker_runtime: String containing shell runtime command for the
            worker instance
        :param key_path: The path to the key used to create EC2 instances and to
            connect to them using paramiko clients
        :param ip_ref: Whether to provide the user with the public IP or private
            IP (useful when configured behind a VPC)
        :param ver: Designated to stamp Security Groups, Placement Groups, keys,
            and all instances launched
        :param purge: Whether to purge previous objects registered to the
            provided version stamp.
        """
        self._kwargs = list(signature(RCluster).parameters.keys())
        self._kwargs.remove('purge')
        self._config = {}
        self._log = getLogger(__name__)
        self.ses = session.Session(
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            region_name=region_name
        )
        self.ec2 = self.ses.resource('ec2')
        if purge:
            _ec2_purge(self.ec2, ver)

        if not key_path:
            self.key_name = ver
            key_path = rcl._set_data('pem')
            kp = self.ec2.create_key_pair(KeyName=ver)
            with open(key_path, 'w') as out:
                out.write(kp.key_material)
        else:
            self.key_name = os.path.splitext(os.path.basename(key_path))[0]

        if 'SecurityGroups' not in instance_conf:
            sg = self.ec2.create_security_group(
                GroupName=ver,
                Description='22 and 8787 open, permissive internal traffic.'
            )
            instance_conf['SecurityGroups'] = [ver]
            sleep(1)  # Security group may not "exist" in time for next call
            sg.authorize_ingress(IpProtocol='tcp', FromPort=22, ToPort=22,
                                 CidrIp='0.0.0.0/0')
            sg.authorize_ingress(IpProtocol='tcp', FromPort=8787, ToPort=8787,
                                 CidrIp='0.0.0.0/0')
            sg.authorize_ingress(SourceSecurityGroupName=ver)

        if 'Placement' not in instance_conf:
            pg = self.ec2.create_placement_group(GroupName=ver,
                                                 Strategy='cluster')
            instance_conf['Placement'] = {'GroupName': ver}

        for key in self._kwargs:
            self.__setattr__(key, locals()[key])