def update_base_image(psinet):
"""
Sets some specific settings for updating the base image.
"""
do_mgr = digitalocean.Manager(
token=psinet._PsiphonNetwork__digitalocean_account.oauth_token)
droplet = make_base_droplet(psinet,
psinet._PsiphonNetwork__digitalocean_account)
update_system_packages(psinet._PsiphonNetwork__digitalocean_account,
droplet.ip_address)
droplet = droplet.load()
droplet = update_kernel(psinet._PsiphonNetwork__digitalocean_account,
do_mgr, droplet)
droplet = take_droplet_snapshot(do_mgr, droplet)
(host, server) = make_psiphon_server(
psinet, psinet._PsiphonNetwork__digitalocean_account, droplet)
if host and server:
failures = transfer_image_to_region(do_mgr, droplet,
droplet.snapshot_ids[-1])
if len(failures) != 0:
print 'There were %s' % len(failures)
else:
if psinet.is_locked:
set_new_base_image(psinet,
psinet._PsiphonNetwork__digitalocean_account,
droplet)
def get_server(digitalocean_account, droplet_id):
try:
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
droplet = do_mgr.get_droplet(droplet_id)
return droplet
except Exception as e:
raise e
def set_new_base_image(psinet, digitalocean_account, droplet):
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
base_image = do_mgr.get_image(digitalocean_account.base_id)
base_image.rename(base_image.name + '_bak')
image = do_mgr.get_image(droplet.snapshot_ids[0])
digitalocean_account.base_id = image.id
psinet.save()
droplet.destroy()
def transfer_server(digitalocean_account, droplet):
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
# transfer image
failures = transfer_image_to_region(do_mgr, droplet,
droplet.snapshot_ids[0])
if len(failures) > 0:
print "Failed to transfer image to regions: %s" % (failures)
return failures
def remove_server(digitalocean_account, droplet_id):
try:
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
droplet = do_mgr.get_droplet(droplet_id)
result = droplet.destroy()
if not result:
raise Exception('Could not destroy droplet: %s' % str(droplet_id))
except Exception as e:
# Don't raise the exception if the server has already been removed
if "The resource you were accessing could not be found." not in str(e):
raise e
def remove_server(digitalocean_account, droplet_id):
"""
Destroys a digitalocean droplet.
**NOTE** : This does not remove the droplet from psinet.
digitalocean_account : DigitalOcean account information
droplet_id : ID of droplet to destroy
"""
try:
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
droplet = do_mgr.get_droplet(droplet_id)
result = droplet.destroy()
if not result:
raise Exception('Could not destroy droplet: %s' % str(droplet_id))
except digitalocean.baseapi.NotFoundError as e:
# Don't raise the exception if the server has already been removed
pass
def remove_server(digitalocean_account, droplet_id):
"""
Destroys a digitalocean droplet.
**NOTE** : This does not remove the droplet from psinet.
digitalocean_account : DigitalOcean account information
droplet_id : ID of droplet to destroy
"""
try:
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
droplet = do_mgr.get_droplet(droplet_id)
result = droplet.destroy()
if not result:
raise Exception('Could not destroy droplet: %s' % str(droplet_id))
except Exception as e:
# Don't raise the exception if the server has already been removed
if "The resource you were accessing could not be found." not in str(e):
raise e
def launch_new_server(digitalocean_account, is_TCS, _, multi_ip=False):
"""
Launches a new droplet and configures it to be a Psiphon server.
This is called from psi_ops.py
digitalocean_account : DigitalOcean account information
returns:
instance of a psinet server
"""
# None TCS id '25617090' only for VPN + filebeat
# Old working None TCS id: 17784624
base_id = '25617090' if not is_TCS else '59795497'
try:
Droplet = collections.namedtuple(
'Droplet', ['name', 'region', 'image', 'size', 'backups'])
new_root_password = psi_utils.generate_password()
new_stats_password = psi_utils.generate_password()
stats_username = digitalocean_account.base_stats_username
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
# Get the base image
base_image = do_mgr.get_image(base_id)
if not base_image:
raise Exception("Base image with ID: %s is not found" % (base_id))
Droplet.image = base_image.id
# Set the default size
droplet_sizes = do_mgr.get_all_sizes()
if not unicode(digitalocean_account.base_size_slug) in [
unicode(s.slug) for s in droplet_sizes
]:
raise 'Size slug not found'
Droplet.size = 's-2vcpu-4gb'
droplet_regions = do_mgr.get_all_regions()
common_regions = list(
set([r.slug for r in droplet_regions
if r.available]).intersection(base_image.regions))
Droplet.region = random.choice(common_regions)
# Use new host_id standard
Droplet.name = str('do-' +
get_datacenter_region(Droplet.region).lower() +
''.join(
random.choice(string.ascii_lowercase)
for x in range(8)))
sshkeys = do_mgr.get_all_sshkeys()
# treat sshkey id as unique
if not unicode(digitalocean_account.ssh_key_template_id) in [
unicode(k.id) for k in sshkeys
]:
raise 'No SSHKey found'
droplet = digitalocean.Droplet(
token=digitalocean_account.oauth_token,
name=Droplet.name,
region=Droplet.region,
image=Droplet.image,
size=Droplet.size,
ssh_keys=[int(digitalocean_account.ssh_key_template_id)],
backups=False)
droplet.create()
if not wait_on_action(do_mgr, droplet, None, 30, 'create',
'completed'):
raise Exception('Event did not complete in time')
droplet = do_mgr.get_droplet(droplet.id)
region = get_datacenter_region(droplet.region['slug'])
datacenter_name = 'Digital Ocean ' + droplet.region['name']
if is_TCS:
set_host_name(digitalocean_account, droplet.ip_address,
new_root_password, droplet.name)
stats_username = psi_utils.generate_stats_username()
set_allowed_users(digitalocean_account, droplet.ip_address,
new_root_password, stats_username)
new_droplet_public_key = refresh_credentials(digitalocean_account,
droplet.ip_address,
new_root_password,
new_stats_password,
stats_username)
assert (new_droplet_public_key)
except Exception as e:
print type(e), str(e)
if droplet is not None:
droplet.destroy()
else:
print type(e), "No droplet to be destroyed: ", str(droplet)
raise
return (droplet.name, is_TCS, 'NATIVE' if is_TCS else None, None,
droplet.id, droplet.ip_address, digitalocean_account.base_ssh_port,
'root', new_root_password,
' '.join(new_droplet_public_key.split(' ')[:2]), stats_username,
new_stats_password, datacenter_name, region, None, None, None,
None, None, None, None, None)
def make_base_droplet(psinet, digitalocean_account):
"""
Updates the base image. This includes installing new system packages
via apt and setting the droplet kernel.
psinet : instance of psinet
digitalocean_account : DigitalOcean account information
droplet_id : digitalocean.Droplet.id to use
droplet_name : (String) to use as droplet name.
droplet_size : (Int) code that denotes the droplet size to use.
Images created with a smaller size can scale
up, but larger sized instances cannot scale down.
test : (Boolean) value to test the new image
"""
droplet = None
try:
if not digitalocean_account:
raise Exception('DigitalOcean account must be provided')
Droplet = collections.namedtuple(
'Droplet', ['name', 'region', 'image', 'size', 'backups'])
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
base_image = do_mgr.get_image(digitalocean_account.base_id)
Droplet.image = base_image.id
Droplet.name = base_image.name
Droplet.size = digitalocean_account.base_size_slug
Droplet.region = base_image.regions[0] if len(
base_image.regions) > 0 else 'nyc1'
sshkeys = do_mgr.get_all_sshkeys()
# treat sshkey id as unique
if not unicode(digitalocean_account.ssh_key_template_id) in [
unicode(k.id) for k in sshkeys
]:
raise 'No SSHKey found'
droplet = digitalocean.Droplet(
token=digitalocean_account.oauth_token,
name=Droplet.name,
region=Droplet.region,
image=Droplet.image,
size=Droplet.size,
ssh_keys=[int(digitalocean_account.ssh_key_template_id)],
backups=False)
droplet.create()
if not wait_on_action(do_mgr,
droplet,
action_id=None,
interval=30,
action_type='create',
action_status='completed'):
raise Exception('Event did not complete in time')
droplet = do_mgr.get_droplet(droplet.id)
if droplet.status != 'active':
result = droplet.power_on()
if not wait_on_action(do_mgr, droplet, result['action']['id'], 30,
'power_on', 'completed'):
raise Exception('Event did not complete in time')
droplet = droplet.load()
except Exception as e:
print type(e), str(e)
return droplet
def get_servers(digitalocean_account):
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
droplets = do_mgr.get_all_droplets()
return [(str(droplet.id), droplet.name) for droplet in droplets
if droplet.tags == []]
def launch_new_server(digitalocean_account, _):
try:
Droplet = collections.namedtuple(
'Droplet', ['name', 'region', 'image', 'size', 'backups'])
new_root_password = psi_utils.generate_password()
new_stats_password = psi_utils.generate_password()
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
# Get the base image
base_image = do_mgr.get_image(digitalocean_account.base_id)
if not base_image:
raise Exception("Base image with ID: %s is not found" %
(digitalocean_account.base_id))
Droplet.image = base_image.id
Droplet.name = str('do-' + ''.join(
random.choice(string.ascii_lowercase) for x in range(8)))
# Set the default size
droplet_sizes = do_mgr.get_all_sizes()
if not unicode(digitalocean_account.base_size_slug) in [
unicode(s.slug) for s in droplet_sizes
]:
raise 'Size slug not found'
Droplet.size = '2gb'
droplet_regions = do_mgr.get_all_regions()
common_regions = list(
set([r.slug for r in droplet_regions
if r.available]).intersection(base_image.regions))
Droplet.region = random.choice(common_regions)
sshkeys = do_mgr.get_all_sshkeys()
# treat sshkey id as unique
if not unicode(digitalocean_account.ssh_key_template_id) in [
unicode(k.id) for k in sshkeys
]:
raise 'No SSHKey found'
droplet = digitalocean.Droplet(
token=digitalocean_account.oauth_token,
name=Droplet.name,
region=Droplet.region,
image=Droplet.image,
size=Droplet.size,
ssh_keys=[int(digitalocean_account.ssh_key_template_id)],
backups=False)
droplet.create()
if not wait_on_action(do_mgr, droplet, None, 30, 'create',
'completed'):
raise Exception('Event did not complete in time')
droplet = do_mgr.get_droplet(droplet.id)
region = get_datacenter_region(droplet.region['slug'])
datacenter_name = 'Digital Ocean ' + droplet.region['name']
new_droplet_public_key = refresh_credentials(digitalocean_account,
droplet.ip_address,
new_root_password,
new_stats_password)
assert (new_droplet_public_key)
except Exception as e:
print type(e), str(e)
if droplet != None:
droplet.destroy()
else:
print type(e), "No droplet to be destroyed: ", str(droplet)
raise
return (droplet.name, None, droplet.id, droplet.ip_address,
digitalocean_account.base_ssh_port, 'root', new_root_password,
' '.join(new_droplet_public_key.split(' ')[:2]),
digitalocean_account.base_stats_username, new_stats_password,
datacenter_name, region, None, None, None, None, None, None)
def update_image(digitalocean_account=None,
droplet_id=None,
droplet_name=None,
droplet_size=None,
test=True):
try:
if not digitalocean_account:
raise Exception('DigitalOcean account must be provided')
Droplet = collections.namedtuple(
'Droplet', ['name', 'region', 'image', 'size', 'backups'])
do_mgr = digitalocean.Manager(token=digitalocean_account.oauth_token)
base_image = do_mgr.get_image(digitalocean_account.base_id)
Droplet.image = base_image.id if not droplet_id else droplet_id
Droplet.name = base_image.name if not droplet_name else droplet_name
Droplet.size = digitalocean_account.base_size_slug if not droplet_size else droplet_size
Droplet.region = base_image.regions[0] if len(
base_image.regions) > 0 else 'nyc1'
sshkeys = do_mgr.get_all_sshkeys()
# treat sshkey id as unique
if not unicode(digitalocean_account.ssh_key_template_id) in [
unicode(k.id) for k in sshkeys
]:
raise 'No SSHKey found'
droplet = digitalocean.Droplet(
token=digitalocean_account.oauth_token,
name=Droplet.name,
region=Droplet.region,
image=Droplet.image,
size=Droplet.size,
ssh_keys=[int(digitalocean_account.ssh_key_template_id)],
backups=False)
droplet.create()
if not wait_on_action(do_mgr,
droplet,
action_id=None,
interval=30,
action_type='create',
action_status='completed'):
raise Exception('Event did not complete in time')
droplet = do_mgr.get_droplet(droplet.id)
if droplet.status != 'active':
result = droplet.power_on()
if not wait_on_action(do_mgr, droplet, result['action']['id'], 30,
'power_on', 'completed'):
raise Exception('Event did not complete in time')
droplet = droplet.load()
update_system_packages(digitalocean_account, droplet.ip_address)
droplet = droplet.load()
result = droplet.reboot()
if not wait_on_action(do_mgr, droplet, result['action']['id'], 30,
'reboot', 'completed'):
raise Exception('Event did not complete in time')
droplet = droplet.load()
if droplet.status == 'active':
result = droplet.shutdown()
if not wait_on_action(do_mgr, droplet, result['action']['id'], 30,
'shutdown', 'completed'):
raise Exception('Event did not complete in time')
droplet = droplet.load()
if droplet.status == 'off':
result = droplet.take_snapshot('psiphon3-template-snapshot')
if not wait_on_action(do_mgr, droplet, result['action']['id'], 120,
'snapshot', 'completed'):
raise Exception('Snapshot took too long to create')
droplet = droplet.load()
if len(droplet.snapshot_ids) < 1:
raise Exception('No snapshot found for image')
# test the server
if test:
result = setup_new_server(digitalocean_account, droplet)
if not result:
raise 'Could not set up server successfully'
# transfer image
failures = transfer_image_to_region(do_mgr, droplet.snapshot_ids[0])
if len(failures) > 0:
print "Failed to transfer image to regions: %s" % (failures)
if psinet.is_locked:
base_image.rename(base_image.name + '_bak')
image = do_mgr.get_image(droplet.snapshot_ids[0])
digitalocean_account.base_id = image.id
psinet.save()
droplet.destroy()
except Exception as e:
print type(e), str(e)
