def get_session():
global session
if not session:
session = boto3.Session(region_name=get_infinity_settings()['aws_region_name'],
profile_name=get_infinity_settings()['aws_profile_name'])
return session
def jupyter(id, print_command_only, local_port, infinity_machine_port):
"""
Setup port forwarding for Jupyter Lab.
This will SSH into the instance and forward the infinity-machine-port to the local machine.
Gives a secure option to connect to your Jupyter Lab instance. Note: You need to run Jupyter Lab
for this command to work.
"""
ec2_resource = get_session().resource('ec2')
instance = ec2_resource.Instance(id)
ssh_private_key_path = get_infinity_settings().get('ssh_private_key_path')
# Ignore the KeyChecking since the hosts are trusted
ssh_command = f"ssh -N -f -oStrictHostKeyChecking=no -oExitOnForwardFailure=yes -i {ssh_private_key_path} " \
f"ubuntu@{instance.public_ip_address} -L localhost:{local_port}:localhost:{infinity_machine_port}"
if print_command_only:
print(ssh_command)
else:
code = os.system(ssh_command)
if code == 0:
print(
f"Port forwarding setup from {instance.public_ip_address}:{infinity_machine_port} -> "
f"localhost:{local_port}")
print(
f"After running Jupyter on the infinity machine, open http://localhost:{local_port}"
)
def price(instance_type):
"""
View the spot and on-demand prices for the instance across all AWS regions.
This command queries the spot instance price for every region that offers the instance.
It also gets the probability that the machine will be preempted. For completeness, it also
shows the on-demand price for comparison.
"""
client = get_session().client('ec2')
response = client.describe_regions()
regions = [item['RegionName'] for item in response['Regions']]
regions.sort()
print("Getting On-demand prices across regions ...")
region_on_demand_price = get_on_demand_instance_pricing(instance_type)
print("Getting frequency of interruptions for spot instances ...")
interruption_frequency = get_interruption_frequency_for_instance_type(instance_type)
print("Getting real-time price of Spot instances ...")
spot_price_table = []
for region in regions:
print_progress()
session = boto3.Session(region_name=region, profile_name=get_infinity_settings()['aws_profile_name'])
client = session.client('ec2')
response = client.describe_availability_zones()
azs = [item['ZoneName'] for item in response['AvailabilityZones']]
spot_price_entries = []
for az in azs:
print_progress()
response = client.describe_spot_price_history(
InstanceTypes=[instance_type],
MaxResults=5,
StartTime=datetime.utcnow(),
EndTime=datetime.utcnow(),
AvailabilityZone=az,
ProductDescriptions=[
"Linux/UNIX",
]
)
spot_price_history = response['SpotPriceHistory']
if not spot_price_history:
continue
spot_price = spot_price_history[0]['SpotPrice']
on_demand_price = region_on_demand_price.get(region, 'N/A')
interrupt_freq = interruption_frequency.get(region, 'N/A')
spot_price_entries.append([region, az, on_demand_price, spot_price, interrupt_freq])
if spot_price_entries:
spot_price_table = spot_price_table + spot_price_entries + ['']
table_header = ['REGION', 'AVAILABILITY ZONE', 'ON-DEMAND PRICE (USD)', 'SPOT PRICE (USD)', 'FREQ OF INTERRUPTION']
print(tabulate(spot_price_table,
headers=table_header,
tablefmt='psql',
colalign=['left', 'left', 'decimal', 'decimal', 'center']))
def track_event(self, event_name, event_properties={}):
user_id = get_infinity_settings().get('user_id_analytics')
if not user_id:
user_id = self.register_user()
self.invoke_track_event(user_id=user_id,
event_name=event_name,
event_properties=event_properties)
def initialize_raven_client():
debug_mode = get_infinity_settings().get('infinity_debug_mode')
if not debug_mode:
# TODO: Sentry prints verbose message on all exceptions
# Infinity throws valid exceptions from boto. So this is distracting
# initialize_sentry()
pass
def get_analytics_client():
global client
if not client:
debug_mode = get_infinity_settings().get('infinity_debug_mode')
if debug_mode:
client = LocalAnalytics()
else:
client = AmplitudeAnalytics()
return client
def quota(instance_type, increase_to):
"""
View and increase quota limit for your account.
This command shows the current quota limit for the instance type in your account.
You will not be able to spin up more than the quota number of instances.
To increase the quota limit, you can request AWS with this command as well.
"""
quota_client = get_session().client('service-quotas')
quota_paginator_client = quota_client.get_paginator('list_service_quotas')
quota_name = f"Running On-Demand {instance_type} instances"
response = quota_paginator_client.paginate(
ServiceCode='ec2',
)
for quota_set in response:
quota_value, quota_code = get_value_for_quota_name(quota_set['Quotas'], quota_name)
if quota_value is not None:
quota_value = int(quota_value)
break
aws_region = get_infinity_settings()['aws_region_name']
if quota_value is None:
print("Cannot find quota for this instance type. Double check if the type value is accurate")
exit(1)
print(f"Your quota limit for {instance_type} in the {aws_region} region is: {quota_value}")
# Get any pending increase request for this instance type:
response = quota_client.list_requested_service_quota_change_history_by_quota(
ServiceCode='ec2',
QuotaCode=quota_code,
)
for request in response['RequestedQuotas']:
if request['Status'] in ['PENDING', 'CASE_OPENED']:
print(f"You have a pending quota request increase to limit: {request['DesiredValue']}")
# Exit if there is an open or pending request, cannot request another one
exit(0)
# Process quota increase now
if increase_to:
if increase_to <= quota_value:
print(f"New quota limit {increase_to} is less than or equal to current limit: {quota_value}")
exit(1)
quota_client.request_service_quota_increase(
ServiceCode='ec2',
QuotaCode=quota_code,
DesiredValue=float(increase_to)
)
print(f"Submitted a quota increase request for {instance_type} in the {aws_region} region to: {increase_to}")
def ssh(id, print_command_only):
"""
SSH into the infinity machine.
This command uses the private key configured in the settings file (~/.infinity/settings.yaml)
to connect.
"""
ec2_resource = get_session().resource('ec2')
instance = ec2_resource.Instance(id)
ssh_private_key_path = get_infinity_settings().get('ssh_private_key_path')
# Ignore the KeyChecking since the hosts are trusted
ssh_command = f"ssh -oStrictHostKeyChecking=no -i {ssh_private_key_path} ubuntu@{instance.public_ip_address}"
if print_command_only:
print(ssh_command)
else:
os.environ["PYTHONUNBUFFERED"] = "1"
os.system(ssh_command)
def register_user(self):
# Get or create new User ID
user_id = get_infinity_settings().get('user_id_analytics')
if not user_id:
user_id = str(uuid.uuid1())
update_infinity_settings(
{
"user_id_analytics": user_id
}
)
# Get user and machine info
user_properties = {
"os_name": platform.system(),
"os_version": platform.release(),
"python_version": platform.python_version(),
"timezone": time.tzname[0],
}
self.invoke_register_user(user_id=user_id, user_properties=user_properties)
return user_id
def teardown():
"""
Deletes all the AWS components created by infinity setup.
This is irrecoverable. It will remove the Cloudformation Stack created
by the setup command. This is the reverse of setup command.
"""
infinity_settings = get_infinity_settings()
session = boto3.Session(region_name=infinity_settings.get('aws_region_name'),
profile_name=infinity_settings.get('aws_profile_name'))
cf_client = session.client('cloudformation')
stack_name = infinity_settings.get('aws_stack_name')
key_name = infinity_settings.get('aws_key_name')
# Check if infinity stack already exists
try:
cf_client.describe_stacks(StackName=stack_name)
except ClientError:
print("Infinity Stack does not exist, nothing to teardown")
exit(1)
if click.confirm("\nAre you sure you want to destroy the Infinity setup? This is irrrecoverable."):
# Delete the stack
cf_client.delete_stack(
StackName=stack_name,
)
print("Stack delete is initiated ...")
# Wait for the stack to be created
waiter = cf_client.get_waiter('stack_delete_complete')
waiter.wait(StackName=stack_name)
print("Infinity Stack deleted successfully")
print("Deleting the public SSH Key")
ec2_client = session.client('ec2')
ec2_client.delete_key_pair(
KeyName=key_name,
)
def attach(volume_id, instance_id):
"""
Attach the volume as a secondary disk to an instance.
This command works only when the instance is running or in stopped state. Ensure that the
instance does not already have a secondary disk volume attached.
"""
ec2_resource = get_session().resource('ec2')
# Check if volume is already attached to a device
volume = ec2_resource.Volume(volume_id)
if volume.attachments:
raise Exception("Volume is currently already attached to an instance")
instance = ec2_resource.Instance(instance_id)
# Check if instance already has another secondary disk
for attachment in instance.block_device_mappings:
if attachment['DeviceName'] == SECONDARY_EBS_DEVICE_NAME:
raise Exception(f"Instance {instance_id} already has a secondary volume attached to it")
# Check if instance is stopped or running
if not instance.state['Name'] in ['stopped', 'running']:
raise Exception(f"Volume can be mounted only to a stopped or running instance. "
f"Current state: {instance.state['Name']}")
# Check if the user data script is finished running
ssh_private_key_path = get_infinity_settings().get('ssh_private_key_path')
if instance.state['Name'] == 'running':
# Ignore the KeyChecking since the hosts are trusted
ssh_command = f"ssh -oStrictHostKeyChecking=no -i {ssh_private_key_path} " \
f"ubuntu@{instance.public_ip_address} ls /etc/load_volume.sh"
code = os.system(ssh_command)
if code != 0:
raise Exception(f"Instance does not have the data mount script. Try again after a few seconds")
print("Attaching volume to the instance...")
volume.attach_to_instance(
Device=SECONDARY_EBS_DEVICE_NAME,
InstanceId=instance_id,
)
ec2_client = get_session().client('ec2')
waiter = ec2_client.get_waiter('volume_in_use')
waiter.wait(VolumeIds=[volume_id])
print("Volume successfully attached to instance")
if instance.state['Name'] == 'running':
print("Mounting the disk to the running machine at: /data")
# Connect to the machine and run /etc/load_volume.sh script
ssh_command = f"ssh -oStrictHostKeyChecking=no -i {ssh_private_key_path} " \
f"ubuntu@{instance.public_ip_address} sudo /etc/load_volume.sh"
os.environ["PYTHONUNBUFFERED"] = "1"
os.system(ssh_command)
response = ec2_client.describe_volumes(
VolumeIds=[
volume_id
]
)
print_volume_info(response['Volumes'])
def create(is_spot, notification_email, instance_type):
"""
Create a new on-demand or spot instance.
Add a notification email address to get notified when the machine unused and running.
Any secondary EBS Volume can be attached after the machine is up and running.
"""
session = get_session()
client = session.client('ec2')
infinity_settings = get_infinity_settings()
# Spot instance request parameters
machine_name_suffix = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
# Pick the machine image
ami = infinity_settings['aws_ami']
if not ami:
print(
"No ami specified in the configuration. Finding the latest Deep learning ami "
)
ami = get_latest_deep_learning_ami()
if not ami:
print(
f"No AMI found, please specify the ami to use in the infinity config file: {CONFIG_FILE_PATH}"
)
exit(1)
instance_type = instance_type or infinity_settings[
'default_aws_instance_type'] or 'p2.xlarge'
print(f"Using ami: {ami}, instance type: {instance_type}")
if is_spot:
instance_market_options = {
'MarketType': 'spot',
'SpotOptions': {
'SpotInstanceType': 'one-time',
}
}
else:
instance_market_options = {}
user_data_file_path = os.path.join(os.path.dirname(__file__),
'user_data.sh')
with open(user_data_file_path, 'r') as f:
user_data = f.read()
response = client.run_instances(
ImageId=ami,
InstanceType=instance_type,
KeyName=infinity_settings.get('aws_key_name'),
BlockDeviceMappings=[{
'DeviceName': '/dev/sda1',
'Ebs': {
'DeleteOnTermination': False,
},
}],
EbsOptimized=True,
SecurityGroupIds=[
infinity_settings.get('aws_security_group_id'),
],
SubnetId=infinity_settings.get('aws_subnet_id'),
MaxCount=1,
MinCount=1,
InstanceMarketOptions=instance_market_options,
UserData=user_data,
TagSpecifications=[{
"ResourceType":
"instance",
"Tags": [{
"Key": "Name",
"Value": f"infinity-{machine_name_suffix}"
}, {
"Key": "type",
"Value": "infinity"
}]
}])
instance_id = response['Instances'][0]['InstanceId']
# Wait until the new instance ID is propagated
sleep(1)
# Get the instance volume id
ec2_resource = session.resource('ec2')
ec2_instance = ec2_resource.Instance(instance_id)
root_volume_id = ec2_instance.block_device_mappings[0]['Ebs']['VolumeId']
# Tag the disk volume
client.create_tags(Resources=[root_volume_id],
Tags=[{
'Key': 'type',
'Value': 'infinity'
}, {
'Key': 'Name',
'Value': f'infinity-{machine_name_suffix}'
}])
# Add email address to SNS Queue and setup alert
if not notification_email:
notification_email = get_infinity_settings().get('notification_email')
if notification_email:
topic_arn = subscribe_email_to_instance_notifications(
session=session, notification_email=notification_email)
create_cloudwatch_alert_for_instance(session=session,
instance_id=instance_id,
topic_arn=topic_arn)
# Wait for the instance to be running again
while ec2_instance.state == 'pending':
print(f"Instance id: {ec2_instance.id}, state: {ec2_instance.state}")
ec2_instance.reload()
print_machine_info([ec2_instance])
print(
"\nHow was your experience setting up your machine? "
"Anything we should improve?: https://github.com/narenst/infinity/issues/new"
)
def setup(region_name, aws_profile, cloud_formation_file, ssh_public_key,
ssh_private_key_path, notification_email):
"""
Sets up infinity before first run.
Before you run Infinity for the first time, you need to run this command. It creates a
CloudFormation stack. And uploads a SSH key to AWS to connect to the instances
you will create. Infinity config is stored in the config file stored at ~/.infinity/settings.yaml.
"""
session = boto3.Session(region_name=region_name, profile_name=aws_profile)
cf_client = session.client('cloudformation')
stack_name = get_infinity_settings().get('aws_stack_name')
key_name = get_infinity_settings().get('aws_key_name')
# Check if infinity stack already exists
try:
stacks = cf_client.describe_stacks(StackName=stack_name)
if stacks:
print("Infinity stack already exists")
except ClientError:
# Error expected if Stack does not exist
# Create a new stack
if not cloud_formation_file:
print(
f"Using default AWS cloudformation from: {CLOUD_FORMATION_FILE_PATH}"
)
if not os.path.exists(CLOUD_FORMATION_FILE_PATH):
copyfile(
os.path.join(os.path.dirname(__file__),
'infinity_cloudformation.yaml'),
CLOUD_FORMATION_FILE_PATH)
cloud_formation_file = CLOUD_FORMATION_FILE_PATH
print(f"Setting up a new Infinity stack in {region_name}")
with open(cloud_formation_file, 'r') as cf_template:
_ = cf_client.create_stack(StackName=stack_name,
TemplateBody=cf_template.read(),
Tags=[{
'Key': 'type',
'Value': 'infinity'
}])
# Wait for the stack to be created
waiter = cf_client.get_waiter('stack_create_complete')
waiter.wait(StackName=stack_name)
print("Infinity stack created successfully")
# Get the configuration from the new stack
stacks_response = cf_client.describe_stacks(
StackName=stack_name)['Stacks'][0]
output_map = {
item['OutputKey']: item['OutputValue']
for item in stacks_response['Outputs']
}
settings_update = {
"aws_region_name": region_name,
"aws_profile_name": aws_profile,
"aws_subnet_id": output_map['InfinitySubnetID'],
"aws_security_group_id": output_map['InfinitySecurityGroupID'],
}
update_infinity_settings(settings_update)
print(
f"Infinity config file is updated with the new stack info: {CONFIG_FILE_PATH}"
)
# Create Key Pair
ec2_client = session.client('ec2')
try:
ec2_client.describe_key_pairs(KeyNames=[key_name])
print("SSH Public key already exists")
except ClientError:
# SSH Key not found, so upload
print("Uploading Public SSH Key")
ec2_client.import_key_pair(KeyName=key_name,
PublicKeyMaterial=ssh_public_key.read())
print("SSH Public Key uploaded")
settings_update = {
"ssh_private_key_path": ssh_private_key_path,
}
update_infinity_settings(settings_update)
print(
f"Infinity config file is updated with the new key info: {CONFIG_FILE_PATH}"
)
# Set notification email
if not notification_email:
notification_email = click.prompt(
'Please enter an email address to send instance notifications from AWS. Press enter to skip this',
type=str,
default=None,
)
if notification_email:
update_infinity_settings({"notification_email": notification_email})
print(
f"Infinity config file is updated with the notification email: {CONFIG_FILE_PATH}"
)
print("\nHow was your setup experience? Please share your feedback here: "
"https://github.com/narenst/infinity/issues/new")
# Register user to Analytics
get_analytics_client().register_user()