def config():
"""
Configure RCluster and AWS EC2 account.
Prompts user for credentials, builds an AMI with specified R packages
installed, and saves out the configuration file with credentials to a hidden
folder in the user's home directory.
"""
import shutil
parser.add_argument('-o', '--outfile', type=str, nargs=1,
default=rcl._set_data('json'),
help='The file in which to save the RCluster' +
'configuration data (stored in JSON format)')
args = parser.parse_args()
logging.basicConfig(level=args.loglevel)
setup_cl = rcl.RCluster.from_config(rcl._get_data('config.json'), purge=True)
setup_cl.write_config(args.outfile)
setup_script = shutil.copyfile(rcl._get_data('ami.sh'), rcl._set_data('sh'))
# TODO: validate inputs
pswd = input("Enter `cluster` user password: "******"Enter R packages to install (in format: `dplyr,plyr,etc`): ")
pkgs = '"' + '", "'.join(pkgs.split(",")) + '"'
with open(setup_script, 'a', newline='') as script:
script.write('echo "cluster:{0}" | chpasswd\n'.format(pswd))
script.write(("R --vanilla -q -e 'install.packages(c({0}), repo = "
"\"https://cran.cnr.berkeley.edu/\")'\n").format(pkgs))
setup_cl.create_ami(setup_fn=setup_script)
setup_cl.write_config(args.outfile)
def terminate():
"""
Terminate all AWS instances associated with the specified RCluster
configuration file.
"""
parser.add_argument('-c', '--config', type=str, nargs=1,
default=rcl._set_data('json'),
help='The JSON RCluster configuration file.')
args = parser.parse_args()
logging.basicConfig(level=args.loglevel)
cluster = rcl.RCluster.from_config(args.config)
cluster.terminate_instances()
def retrieve_cluster():
"""
Retrieve the access IP address of the current manager instance (if live).
Also opens a browser to the manager's RStudio Server.
"""
parser.add_argument('-c', '--config', type=str, nargs=1,
default=rcl._set_data('json'),
help='The JSON RCluster configuration file.')
args = parser.parse_args()
logging.basicConfig(level=args.loglevel)
log = logging.getLogger()
cluster = rcl.RCluster.from_config(args.config)
ip = cluster.get_manager_ip()
if ip:
log.debug(ip)
_open_ip(ip)
def main():
"""Launch an RCluster using the information saved to a configuration file"""
parser.add_argument('-w', '--workers', type=int, nargs=1, default=[1],
help='The number of workers to launch.')
parser.add_argument('-t', '--type', type=str, nargs=1, default='m4.large',
help='The instance type to use.')
parser.add_argument('-c', '--config', type=str, nargs=1,
default=rcl._set_data('json'),
help='The JSON RCluster configuration file.')
args = parser.parse_args()
logging.basicConfig(level=args.loglevel)
log = logging.getLogger()
try:
cluster = rcl.RCluster.from_config(args.config)
except FileNotFoundError as err:
log.err('Run `rcluster-config`, first, to generate your own config',
'file with the minimum necessary data to start an R cluster.')
raise err
ip = cluster.get_manager_ip()
if ip:
log.info("Active rcluster found.\n",
"Run `rcluster-terminate` to remove the previous cluster.\n"
"Returning current manager instance.\n")
else:
cluster.create_cluster(args.workers[0], InstanceType=args.type)
ip = cluster.access_ip
_open_ip(ip)
cl_data = ('Manager IP Address:', ip)
log.info(cl_data)
term = ''
while term not in 'yn':
term = input("""
Type 'y' to terminate the cluster and exit.
Type 'n' to exit without terminating the cluster.
You can always terminate your current RCluster by running
`rcluster-terminate` from the command line.
""")
if term == 'y':
cluster.terminate_instances()
def __init__(self, aws_access_key_id, aws_secret_access_key, region_name,
instance_conf, manager_runtime=None, worker_runtime=None,
key_path=None, ip_ref='public_ip_address', ver=rcl.__ver__,
purge=False):
"""Initialize the RCluster object.
:param aws_access_key_id: AWS access key provided to
boto3.session.Session()
:param aws_secret_access_key: AWS secret access key provided to
boto3.session.Session()
:param region_name: The accessibility region provided to
boto3.session.Session()
:param instance_conf: Dictionary defining {'ami': '', 'type': ''} for
instances (where 'ami' is the AMI ID for the instances and type is
the instance type used); can also contain other parameters to
boto3's EC2.ServiceResource.create_instances
:param manager_runtime: String containing shell runtime command for the
manager instance
:param worker_runtime: String containing shell runtime command for the
worker instance
:param key_path: The path to the key used to create EC2 instances and to
connect to them using paramiko clients
:param ip_ref: Whether to provide the user with the public IP or private
IP (useful when configured behind a VPC)
:param ver: Designated to stamp Security Groups, Placement Groups, keys,
and all instances launched
:param purge: Whether to purge previous objects registered to the
provided version stamp.
"""
self._kwargs = list(signature(RCluster).parameters.keys())
self._kwargs.remove('purge')
self._config = {}
self._log = getLogger(__name__)
self.ses = session.Session(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=region_name
)
self.ec2 = self.ses.resource('ec2')
if purge:
_ec2_purge(self.ec2, ver)
if not key_path:
self.key_name = ver
key_path = rcl._set_data('pem')
kp = self.ec2.create_key_pair(KeyName=ver)
with open(key_path, 'w') as out:
out.write(kp.key_material)
else:
self.key_name = os.path.splitext(os.path.basename(key_path))[0]
if 'SecurityGroups' not in instance_conf:
sg = self.ec2.create_security_group(
GroupName=ver,
Description='22 and 8787 open, permissive internal traffic.'
)
instance_conf['SecurityGroups'] = [ver]
sleep(1) # Security group may not "exist" in time for next call
sg.authorize_ingress(IpProtocol='tcp', FromPort=22, ToPort=22,
CidrIp='0.0.0.0/0')
sg.authorize_ingress(IpProtocol='tcp', FromPort=8787, ToPort=8787,
CidrIp='0.0.0.0/0')
sg.authorize_ingress(SourceSecurityGroupName=ver)
if 'Placement' not in instance_conf:
pg = self.ec2.create_placement_group(GroupName=ver,
Strategy='cluster')
instance_conf['Placement'] = {'GroupName': ver}
for key in self._kwargs:
self.__setattr__(key, locals()[key])