def start_digitalocean_ingestion(neo4j_session: neo4j.Session,
config: Config) -> None:
"""
If this module is configured, perform ingestion of DigitalOcean data. Otherwise warn and exit
:param neo4j_session: Neo4J session for database interface
:param config: A cartography.config object
:return: None
"""
if not config.digitalocean_token:
logger.info(
'DigitalOcean import is not configured - skipping this module. See docs to configure.'
)
return
common_job_parameters = {
"UPDATE_TAG": config.update_tag,
}
manager = Manager(config.digitalocean_token)
"""
Get Account ID related to this credentials and pass it along in `common_job_parameters` to avoid cleaning up other
accounts resources
"""
account = manager.get_account()
common_job_parameters["DO_ACCOUNT_ID"] = account.uuid
platform.sync(neo4j_session, account, config.update_tag,
common_job_parameters)
project_resources = management.sync(neo4j_session, manager,
config.update_tag,
common_job_parameters)
compute.sync(neo4j_session, manager, project_resources, config.update_tag,
common_job_parameters)
return
def create_droplets():
"""
Create three new DigitalOcean droplets -
node-1, node-2, node-3
"""
manager = Manager(token=DIGITAL_OCEAN_ACCESS_TOKEN)
# Get ALL SSH keys
keys = manager.get_all_sshkeys()
# Get single SSH key
# all_keys = manager.get_all_sshkeys()
# keys = []
# for key in all_keys:
# if key.name == '<ADD_YOUR_KEY_NAME_HERE>':
# keys.append(key)
for num in range(3):
node = f'node-{num + 1}'
droplet = Droplet(
token=DIGITAL_OCEAN_ACCESS_TOKEN,
name=node,
region='nyc3',
image='ubuntu-16-04-x64',
size_slug='4gb',
tags=[node],
ssh_keys=keys,
)
droplet.create()
print(f'{node} has been created.')
def get_roles(client_id=None, api_key=None, blacklist=None, ssh_port=22):
ip_blacklist = blacklist or []
client_id = client_id or os.getenv("DO_CLIENT_ID")
api_key = api_key or os.getenv("DO_API_KEY")
if not client_id or not api_key:
print("You have to provide the client ID and API key for Digital "
"Ocean. Set DO_CLIENT_ID and DO_API_KEY environment variables.")
sys.exit(28)
if not env.server_name_regex:
env.server_name_regex = re.compile(r'(?P<role>.+)')
if not env.server_format:
env.server_format = "{ip}:{port}"
# Retrieve the app server IPs from the DO API
manager = Manager(client_id=client_id, api_key=api_key)
roles = {}
for droplet in manager.get_all_droplets():
if droplet.ip_address in ip_blacklist:
continue
match = env.server_name_regex.match(droplet.name)
if not match:
continue
roles.setdefault(match.group('role'), []).append(
env.server_format.format(ip=droplet.ip_address, port=ssh_port))
return roles
def destroy_droplets(ctx):
"""Destroy the droplets - node-1, node-2, node-3"""
manager = Manager(token=DIGITAL_OCEAN_ACCESS_TOKEN)
for num in range(3):
node = f"node-{num + 1}"
droplets = manager.get_all_droplets(tag_name=node)
for droplet in droplets:
droplet.destroy()
print(f"{node} has been destroyed.")
def get_first_droplet_by_tag(tag):
token = load_config()['token']
mngr = Manager(token=token)
droplet = mngr.get_all_droplets(tag_name=tag)
if droplet:
droplet = droplet[0]
print('droplet "%s"(%s): ip=%s' % (droplet.name, droplet.id, droplet.ip_address))
return droplet
return None
def list(item_type):
token = os.getenv('access_token')
manager = Manager(token=token)
if item_type == 'droplets':
droplets = manager.get_all_droplets()
result = []
for droplet in droplets:
result.append({'id': droplet.id, 'name': droplet.name,
'status': droplet.status})
click.echo(json.dumps(result))
def find_droplet_by_name(droplet_name: str, access_token: str) -> Droplet:
"""
Find droplet by name. Finds the first one by name, I think from
any project.
:param droplet_name: The name of the droplet to find.
:param manager: A Manager object to use.
"""
manager = Manager(token=access_token)
try:
return next(droplet for droplet in manager.get_all_droplets()
if droplet.name == droplet_name)
except StopIteration:
raise DropletNotFound(f"Droplet {droplet_name} not found.") from None
def _create_ssh_keys(manager: digitalocean.Manager) -> List[digitalocean.SSHKey]:
"""
Upload a ssh public key to Digital Ocean, if not already loaded.
If it is already loaded, then just get the id.
:return: List of SSHkey objects.
"""
registered_keys = manager.get_all_sshkeys()
ssh_keys = []
print('Registering SSH keys...')
print('Known keys:')
print('\n'.join([key.name for key in registered_keys]))
key = DIGITAL_OCEAN_PUBLIC_KEY
print(f'Checking key: {key}')
with open(os.path.join(SSH_PATH, f"{key}")) as file:
public_key = file.read().rstrip('\n')
registered = [rkey for rkey in registered_keys if rkey.public_key == public_key]
for rkey in registered_keys:
if rkey.public_key == public_key:
ssh_keys.append(registered[0])
print(
f'SSH key {key} verified already registered as '
f'{registered[0].name}.')
break
else:
key = digitalocean.SSHKey(
name=key,
public_key=public_key,
token=DIGITAL_OCEAN_TOKEN,
)
key.create()
ssh_keys.append(key)
print(f'SSH key {key} has now been registered with name {key.name}')
return ssh_keys
def _get_droplet_by_name(
name: str, manager: digitalocean.Manager, fail_if_missing=True
) -> digitalocean.Droplet:
droplets = manager.get_all_droplets()
droplet: digitalocean.Droplet = next((d for d in droplets if d.name == name), None)
if not droplet and fail_if_missing:
click.secho(f"No droplet found for name: {name}", fg="red")
sys.exit(1)
return droplet
def get_remote_tags(manager: Manager) -> list:
"""
API call to DigitalOcean's API v2 in order to fetch all remote tags.
:param digitalocean.Manager.Manager manager: instance
:return: list of digitalocean.Tag.Tag instances
"""
with pong(text=cyan_text("Retrieving existing tags...")):
return manager.get_all_tags()
def get_account_data(manager: Manager) -> Account:
"""
API call to DigitalOcean's API v2 in order to get account data.
:param digitalocean.Manager.Manager manager: instance
:return: digitalocean.Account.Account instance
"""
with pong(text=cyan_text("Retrieving account data...")):
return manager.get_account()
def get_remote_ssh_keys(manager: Manager) -> list:
"""
API call to DigitalOcean's API v2 in order to fetch ssh keys.
:param digitalocean.Manager.Manager manager: instance
:return: list of digitalocean.SSHKey.SSHKey instances
"""
with pong(text=cyan_text("Retrieving public keys from your account...")):
return manager.get_all_sshkeys()
def get_all_sizes(manager: Manager) -> list:
"""
API call to DigitalOcean's API v2 in order to fetch all sizes.
:param digitalocean.Manager.Manager manager: instance
:return: list of digitalocean.Size.Size instances
"""
with pong(text=cyan_text("Retrieving available sizes...")):
return manager.get_all_sizes()
def login():
if request.args.get('code') is None:
abort(404)
# Get from Chrome extension
token = request.args.get('code')
manager = Manager(token=token)
# Instantiate ``api`` object to setup authentication for DO API.
my_droplets = manager.get_all_droplets()
# Check for success
print(my_droplets)
user_ssh_key = request.args.get('ssh')
key = SSHKey(token='bb7f9e5b82a17b7304efde1b9cd886fc329f09340fa172c3c27d890b099c25cb',
name='uniquehostname',
public_key=user_ssh_key)
# key is created succesfully.
key.create()
return "Login Success"
def create_droplets(ctx):
"""
Create three new DigitalOcean droplets -
node-1, node-2, node-3
"""
manager = Manager(token=DIGITAL_OCEAN_ACCESS_TOKEN)
keys = manager.get_all_sshkeys()
for num in range(3):
node = f"node-{num + 1}"
droplet = Droplet(
token=DIGITAL_OCEAN_ACCESS_TOKEN,
name=node,
region="nyc3",
image="ubuntu-20-04-x64",
size="s-2vcpu-4gb",
tags=[node],
ssh_keys=keys,
)
droplet.create()
print(f"{node} has been created.")
def _get_current_droplets(manager: digitalocean.Manager) -> List[str]:
"""
Get a list of the current droplets to make sure the same name will not
be used twice.
:return: List of current droplet names.
"""
print("Gathering information on current Digital Ocean droplets...")
droplets = manager.get_all_droplets()
names = [x.name for x in droplets]
return names
def create_droplets(ctx):
"""
Create three new DigitalOcean droplets -
node-1, node-2, node-3
"""
manager = Manager(token=DIGITAL_OCEAN_ACCESS_TOKEN)
keys = manager.get_all_sshkeys()
for num in range(3):
node = f'node-{num + 1}'
droplet = Droplet(
token=DIGITAL_OCEAN_ACCESS_TOKEN,
name=node,
region='nyc3',
image='ubuntu-20-04-x64',
size_slug='4gb',
tags=[node],
ssh_keys=keys,
)
droplet.create()
print(f'{node} has been created.')
class DigitalDroplets:
def __init__(self, token):
print("[+] All the shit will be delete !!! ")
self.Manager = Manager(token=token)
def delete_droplets(self):
print('[+] Droplets : ')
self.__delete(self.Manager.get_all_droplets())
def delete_ssh(self):
print('[+] SSH Keys : ')
self.__delete(self.Manager.get_all_sshkeys())
def delete_snapshots(self):
print('[+] Snapshot : ')
self.__delete(self.Manager.get_all_snapshots())
@staticmethod
def __delete(what_to_delete):
for item in what_to_delete:
print(" Deleting {} : {}".format(item.name, item.destroy()))
def update_records(records: List[Record], ipv4: str):
manager = Manager()
domains = manager.get_all_domains()
for record in records:
assert record.ttl >= 30, 'Unable to set a ttl lower than 30.'
assert record.type == 'A', 'Only A records (IPv4) are currently supported.'
domain = None
sub_domain = None
for domain in domains:
if not record.name.endswith(domain.name):
continue
sub_domain_raw, _ = record.name.split(domain.name)
if sub_domain_raw:
sub_domain = sub_domain_raw.strip('.')
else:
sub_domain = '@'
break
if not sub_domain:
raise ValueError(
f'Unable to find find registered domain name for record '
f'{record!r}')
do_records = domain.get_records()
do_records_filtered = [
r for r in do_records
if r.name == sub_domain and r.type == record.type
]
if not do_records_filtered:
raise ValueError(
f'Unable to find record {sub_domain!r} for domain '
f'{domain!r}')
do_record = do_records_filtered[0]
if do_record.data == ipv4 and do_record.ttl == record.ttl:
click.echo(
f'Record {record.name!r} is up to date. Skipping update.')
continue
do_record.data = ipv4
do_record.ttl = record.ttl
do_record.save()
click.echo(
f'Updated {record.name!r} with ip {ipv4} and ttl {record.ttl}')
def get_addresses(ctx, type):
"""Get IP address"""
manager = Manager(token=DIGITAL_OCEAN_ACCESS_TOKEN)
if type == "master":
droplet = manager.get_all_droplets(tag_name="node-1")
print(droplet[0].ip_address)
hosts.append(droplet[0].ip_address)
elif type == "workers":
for num in range(2, 4):
node = f"node-{num}"
droplet = manager.get_all_droplets(tag_name=node)
print(droplet[0].ip_address)
hosts.append(droplet[0].ip_address)
elif type == "all":
for num in range(3):
node = f"node-{num + 1}"
droplet = manager.get_all_droplets(tag_name=node)
print(droplet[0].ip_address)
hosts.append(droplet[0].ip_address)
else:
print('The "type" should be either "master", "workers", or "all".')
print(f"Host addresses - {hosts}")
def get_addresses(type):
""" Get IP address """
manager = Manager(token=DIGITAL_OCEAN_ACCESS_TOKEN)
if type == 'master':
droplet = manager.get_all_droplets(tag_name='node-1')
print(droplet[0].ip_address)
env.hosts.append(droplet[0].ip_address)
elif type == 'workers':
for num in range(2, 4):
node = f'node-{num}'
droplet = manager.get_all_droplets(tag_name=node)
print(droplet[0].ip_address)
env.hosts.append(droplet[0].ip_address)
elif type == 'all':
for num in range(3):
node = f'node-{num + 1}'
droplet = manager.get_all_droplets(tag_name=node)
print(droplet[0].ip_address)
env.hosts.append(droplet[0].ip_address)
else:
print('The "type" should be either "master", "workers", or "all".')
print(f'Host addresses - {env.hosts}')
def get_all_images(manager: Manager,
image_type: str = "distribution",
text="") -> list:
"""
API call to DigitalOcean's API v2 in order to fetch available images per
the selected image type.
:param digitalocean.Manager.Manager manager: instance
:param str image_type: "distribution" or "application" or "all"
:param str text: a text to appear while fetching
:return: list of digitalocean.Image.Image instances
"""
text = text or f"Retrieving {image_type.lower()} images..."
if image_type == "all":
image_type = None
with pong(text=cyan_text(text)):
return manager.get_images(type=image_type)
def add_regions(snap):
print("[+] Adding Regions {}: ".format(snap.name))
print(" Current regions : {} ".format(" ".join(snap.regions)))
available_regions = Manager(token=token).get_all_regions()
print(" Available regions : {} ".format(" ".join(
[v.slug for v in available_regions])))
if set(snap.regions) == set([region.slug for region in available_regions]):
print(" All Available Regions already Added ")
return
new_regions = [
new for new in available_regions if (new.slug not in snap.regions)
]
for region in new_regions:
print(" - Adding {} : {} ".format(region.name, region.slug),
end=" ... ")
try:
adding = snap.get_data("images/%s/actions/" % snap.id,
type='POST',
params={
"type": "transfer",
"region": region.slug
})
# get_data returns a dict not actions to use for waiting
# TODO Get transfer status
# adding.wait()
print(" Done")
except DataReadError:
print("This image has already been transfered to this region")
continue
print("[!] wait for region transfer not implemented yet")
print("[!] you should wait for transfer to finish ")
def __init__(self, kwargs):
#configure the resource via credentials
self.access_token = kwargs['access_token']
self.manager = Manager(token=self.access_token)
self.region = kwargs['region']
return super().__init__()
# Instantiate ``api`` object to setup authentication for DO API.
my_droplets = manager.get_all_droplets()
# Check for success
print(my_droplets)
user_ssh_key = request.args.get('ssh')
key = SSHKey(token='bb7f9e5b82a17b7304efde1b9cd886fc329f09340fa172c3c27d890b099c25cb',
name='uniquehostname',
public_key=user_ssh_key)
# key is created succesfully.
key.create()
return "Login Success"
token = 'bb7f9e5b82a17b7304efde1b9cd886fc329f09340fa172c3c27d890b099c25cb'
repo_url = 'https://github.com/CapsLockHacks/dockerfile_test'
manager = Manager(token=token)
@app.route('/create', methods=['GET'])
@cross_origin(origin='*',headers=['Content-Type','Authorization'])
def create():
user = getpass.getuser()
user_ssh_key = open('/home/{}/.ssh/id_rsa.pub'.format(user)).read()
key = SSHKey(token='bb7f9e5b82a17b7304efde1b9cd886fc329f09340fa172c3c27d890b099c25cb',
name='uniquehostname',
public_key=user_ssh_key)
try:
key.create()
except:
print("[!] wait for region transfer not implemented yet")
print("[!] you should wait for transfer to finish ")
if __name__ == '__main__':
for token in tokens:
start = timer()
# step 1 : created Droplet
droplet = create_droplet(token)
sleep(10) # to make sure it's up
# step 2 : Connect to droplet
ssh_session = connect_to_droplet(droplet.ip_address)
# step 3 : run commands
run_cmd(ssh_session)
# step 4 : create snapshot
snap = create_snapshot(droplet)
# step 5 : add all regions
add_regions(snap)
# in case of errors in add_regions let's iterate over all existing snapshots and add regions , makhasrin walo
for snap in Manager(token=token).get_all_snapshots():
add_regions(snap)
print("[+] Total time {}".format(timer() - start))
def get_droplet_status(node):
"""Given a droplet's tag name, return the status of the droplet"""
manager = Manager(token=DIGITAL_OCEAN_ACCESS_TOKEN)
droplet = manager.get_all_droplets(tag_name=node)
return droplet[0].status
def get_droplet_list():
manager = Manager(token=DO_TOKEN)
droplets = manager.get_all_droplets(tag_name=DO_TAGS)
return droplets
def get_droplets(self):
"""Get active droplets"""
manager = Manager(token=self.api_token)
my_droplets = manager.get_all_droplets()
return my_droplets
def get_projects(manager: Manager) -> list:
return manager.get_all_projects()
import os
from digitalocean import Manager, Droplet
from time import sleep
from time import time
import paramiko
connection = Manager(token=os.environ["DO_TOKEN"])
key = []
key.append(connection.get_ssh_key("18830428"))
for i in connection.get_my_images():
if i.name == os.environ["IMAGE_NAME"]:
image = connection.get_image(i.id).id
break
srv = Droplet(token=os.environ["DO_TOKEN"], image=image, size="s-1vcpu-1gb", region="fra1", ssh_keys=key, \
name=os.environ["TRAVIS_COMMIT"], user_data="#!/bin/bash\nsed -i 's%PasswordAuthentication no%PasswordAuthentication yes%g' /etc/ssh/sshd_config\nservice sshd restart\n")
start = time()
srv.create()
for i in range(1, 12):
if srv.get_actions()[0].status == 'completed':
stop = time()
print(
"Instance provisioninng took {}. Saving envs to run_instance_envs."
.format(stop - start))
start = time()
ip_addr = Droplet.get_object(api_token=os.environ["DO_TOKEN"],
droplet_id=srv.id).ip_address
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
while (True):
try:
client.connect(ip_addr,
