def create_security_group(self):
regions = self.protocol_config['CloudProviders']['scaleway']['regions']
for idx in range(len(regions)):
api = ComputeAPI(auth_token=self.token, region=regions[idx])
api.query().security_groups.post({
'organization': self.account_id,
'name': 'MatrixSG%s' % regions[idx],
'description': 'Default security group for MATRIX system'
})
def stop_instances(self):
protocol_name = self.protocol_config['protocol']
regions = self.protocol_config['CloudProviders']['scaleway']['regions']
for region in regions:
api = ComputeAPI(region=region, auth_token=self.token)
servers = self.describe_instances(region, protocol_name)
for server in servers:
if server['state'] == 'running':
server_id = server['id']
api.query().servers(server_id).action.post({'action': 'poweroff'})
def describe_instances(self, region_name, machines_name):
api = ComputeAPI(region=region_name, auth_token=self.token)
servers = api.query().servers.get()
instances = []
for server in servers['servers']:
if server['hostname'] == machines_name:
instances.append(server)
return instances
def deploy_instances(self):
protocol_name = self.protocol_config['protocol']
machine_type = self.protocol_config['CloudProviders']['scaleway']['instanceType']
regions = self.protocol_config['CloudProviders']['scaleway']['regions']
number_of_parties = self.protocol_config['CloudProviders']['scaleway']['numOfParties']
number_duplicated_servers = 0
if len(regions) > 1:
number_of_instances = number_of_parties // len(regions)
if number_of_parties % len(regions):
number_duplicated_servers = number_of_parties % len(regions)
else:
number_of_instances = number_of_parties
for idx in range(len(regions)):
api = ComputeAPI(auth_token=self.token, region=regions[idx])
number_of_instances_to_deploy = self.check_running_instances(regions[idx], machine_type)
if idx < number_duplicated_servers:
number_of_instances_to_deploy = (number_of_instances - number_of_instances_to_deploy) + 1
else:
number_of_instances_to_deploy = number_of_instances - number_of_instances_to_deploy
print('Deploying instances :\nregion : %s\nnumber of instances : %s\ninstance_type : %s\n'
% (regions[idx], number_of_instances_to_deploy, machine_type))
for idx2 in range(number_of_instances_to_deploy):
try:
res = api.query().servers.post({
'name': protocol_name,
'organization': self.account_id,
'image': 'accc8537-84fa-4cd8-b4d4-dd35388c7165',
'commercial_type': machine_type
})
server_id = res['server']['id']
api.query().servers(server_id).action.post({'action': 'poweron'})
except Exception as e:
print(e.__traceback__)
time.sleep(number_of_instances_to_deploy * 15)
self.get_network_details()
def terminate_instances(self):
self.stop_instances()
protocol_name = self.protocol_config['protocol']
regions = self.protocol_config['CloudProviders']['scaleway']['regions']
number_of_instances = self.protocol_config['CloudProviders']['scaleway']['numOfParties']
# Don't change the sleep duration, trust me I'm engineer
time.sleep(number_of_instances * 15)
for region in regions:
api = ComputeAPI(region=region, auth_token=self.token)
servers = self.describe_instances(region, protocol_name)
for server in servers:
if server['state'] == 'stopped' or server['state'] == 'stopped in place':
server_id = server['id']
volume_id = server['volumes']['0']['id']
# delete ip
if not server['public_ip'] is None:
ip_id = server['public_ip']['id']
api.query().ips(ip_id).delete()
# delete server
api.query().servers(server_id).delete()
# delete volume
api.query().volumes(volume_id).delete()
def get_servers(self):
'''
query scaleway api and pull down a list of servers
'''
self.parse_config()
api = ComputeAPI(auth_token=self.auth_token)
result = api.query().servers.get()
self.inventory = [[i['name'], i['public_ip'], i['tags']]
for i in result['servers']]
for host, ip_info, tags in self.inventory:
if ip_info:
self.response['_meta']['hostvars'][host] = {
'ansible_host': ip_info['address']
}
if tags:
for tag in tags:
if tag.startswith('group:'):
self._add_to_response(tag.split(':')[1], host)
class Scaleway:
conn = None
def __init__(self, cli_args = None):
# Create Scaleway API
self.conn = ComputeAPI(auth_token = cli_args.scaleway_key)
def get_domains(self):
# dictionary for result
result = {}
servers = self.conn.query().servers.get()
for key, value in servers.iteritems():
for server in value:
domain_name = str(server['name'])
result[domain_name] = {}
result[domain_name]['id'] = str(server['id'])
result[domain_name]['state'] = True if server['state'] == 'running' else False
result[domain_name]['name'] = domain_name
result[domain_name]['memory'] = 2097152 if str(server['commercial_type']) == 'C1' else 0
result[domain_name]['vcpu'] = 2 if str(server['commercial_type']) == 'C1' else 0
result[domain_name]['ip_assignment'] = [server['public_ip']['address'], server['private_ip']]
result[domain_name]['hostname'] = 'www.scaleway.com'
result[domain_name]['hypervisor'] = 'Scaleway'
result[domain_name]['uri'] = 'https://www.scaleway.com/'
return result
def domain_start(self, name):
return 501
def domain_stop(self, name):
return 501
def domain_shutdown(self, name):
return 501
def domain_restart(self, name):
return 501
def domain_reboot(self, name):
return 501
# Management for Scaleway servers
from scaleway.apis import ComputeAPI
with open("real_org.id", "r") as org_file:
ORG_ID = org_file.read()
with open("real.key", "r") as key_file:
api = ComputeAPI(region='par1', auth_token=key_file.read())
all_srv = api.query().servers.get()["servers"]
# content of each server in list, note most are yet more dicts:
# dict_keys(['allowed_actions', 'maintenances', 'state_detail', 'image', 'creation_date', 'public_ip', 'private_ip',
# 'id', 'dynamic_ip_required', 'modification_date', 'enable_ipv6', 'hostname', 'state', 'bootscript', 'location',
# 'boot_type', 'ipv6', 'commercial_type', 'tags', 'arch', 'extra_networks', 'compute_cluster', 'name', 'protected',
# 'volumes', 'security_group', 'organization'])
def show_detail(server_id: str = None):
"""Print basic details of server given an id, or all servers if no id given."""
def __print_srv__(target_srv):
print(target_srv["hostname"] + " @ " +
target_srv["public_ip"]["address"])
print(target_srv["id"])
print(target_srv["state_detail"])
print("actions:", target_srv["allowed_actions"])
print("#####")
if id is not None:
for srv in all_srv:
class Manager:
"""
Root of all evil. Instances of Server, IP, etc. should all be created and referenced using commands here.
"""
def __init__(self, auth_id, organization_id, region):
self.api = ComputeAPI(region=region, auth_token=auth_id)
self.org_id = organization_id
self.servers = set()
self.ips = set()
self.volumes = set()
self.build_ip_list()
self.build_server_list()
self.build_volume_list()
def build_server_list(self):
"""
Repopulate the set of Server instances. Will destroy any existing set and all references to it,
reloading information from the Scaleway API. Accessible from Manager.servers
"""
self.servers = set()
servers_dict = self.api.query().servers.get()["servers"]
for server in servers_dict:
self.servers.add(Server(self, server))
def build_ip_list(self):
"""
Repopulate the set of IP instances. Will destroy any existing set and all references to it,
reloading information from the Scaleway API. Accessible from Manager.ips
"""
self.ips = set()
ips_dict = self.api.query().ips().get()
for ip in ips_dict['ips']:
self.ips.add(IP(self, ip))
def build_volume_list(self):
"""
Repopulate the set of Volume instances. Will destroy any existing set and all reference to it,
reloading information from the Scaleway API. Accessible from Manager.volumes
"""
self.volumes = set()
volumes_dict = self.api.query().volumes().get()
for volume in volumes_dict["volumes"]:
self.volumes.add(Volume(self, volume))
def get_server_by_name(self, name: str):
for server in self.servers:
if server.name == name:
return server
raise LookupError("No server with that name found!")
def get_server_by_id(self, id_str: str):
for server in self.servers:
if server.id == id_str:
return server
raise LookupError("No server with that ID found!")
def get_ip_by_id(self, id_str: str):
"""
Obtain an IP from a known ID string.
"""
for ip in self.ips:
if id_str == ip.id:
return ip
raise LookupError("No IP with that ID found!")
def get_ip_by_address(self, address: str):
"""
Obtain an IP from a known public address.
"""
for ip in self.ips:
if address == ip.address:
return ip
raise LookupError("No IP with that address found!")
def new_ip(self):
"""Creates a brand new IP through the Scaleway API and creates an associated IP instance.
Returns that instance."""
result = self.api.query().ips.post({"organization": self.org_id})['ip']
print(result)
new_ip = IP(self, result)
self.ips.add(new_ip)
return new_ip
class ScalewayInstances:
def __init__(self, args):
self.log = logging.getLogger(self.__class__.__name__)
if args.errclnup:
atexit.register(self.cleanup)
self._args = args
self._images = {}
self.instances = []
if not args.auth_token or not args.org_id:
log.fatal('auth-token and org-id are required parameters')
sys.exit(1)
self._api = ComputeAPI(auth_token=self._args.auth_token, region=self._args.region)
@staticmethod
def add_args(p):
p.add_argument('--org-id ', help='Scaleway Organization ID', dest='org_id', type=str,
default=os.environ.get('SCALEWAY_ORG_ID', None))
p.add_argument('--auth-token', help='Scaleway Auth Token', dest='auth_token', type=str,
default=os.environ.get('SCALEWAY_AUTH_TOKEN', None))
p.add_argument('--region', help='Scaleway Region (default: par1)', dest='region', type=str, default='par1')
p.add_argument('--id', help='Cluster Identifier (default: random)', dest='cid', type=str,
default=str(uuid.uuid4()))
return p
@property
def images(self):
if len(self._images) == 0:
self.log.debug('Fetching OS Images from Scaleway API')
for image in self._api.query().images.get()['images']:
if image['arch'] == 'x86_64':
self.log.debug(
'Found image {} with id {} ({})'.format(image['name'], image['id'], image['creation_date']))
self._images[image['name']] = image
return self._images
@retry(stop_max_attempt_number=3, wait_exponential_multiplier=1000, wait_exponential_max=30000,
retry_on_exception=retry_on_apierror)
def server(self, sid):
try:
resp = self._api.query().servers(sid).get()
return resp['server']
except slumber.exceptions.HttpClientError as e:
pprint(e.content)
raise
def volume(self, vid):
try:
resp = self._api.query().volumes(vid).get()
return resp['volume']
except slumber.exceptions.HttpClientError as e:
pprint(e.content)
raise
def create_server(self, name, image, commercial_type, volumes, tags=[], enable_ipv6=False):
self.log.info("Creating server {} of type {} from image {}".format(name, commercial_type, image))
try:
resp = self._api.query().servers().post({'organization': self._args.org_id,
'name': name,
'image': image,
'volumes': volumes,
'commercial_type': commercial_type,
'tags': tags,
'enable_ipv6': enable_ipv6})
return resp['server']
except slumber.exceptions.HttpClientError as e:
pprint(e.content)
raise
def create_volume(self, name, size, volume_type='l_ssd'):
self.log.info("Creating volume {} with size {}".format(name, size))
try:
resp = self._api.query().volumes().post({'name': name,
'organization': self._args.org_id,
'size': size,
'volume_type': volume_type})
return resp['volume']
except slumber.exceptions.HttpClientError as e:
pprint(e.content)
raise
def delete_server(self, sid):
self.log.info("Deleting server with ID {}".format(sid))
self.wait_for(sid, 'stopped')
try:
resp = self._api.query().servers(sid).delete()
return resp
except slumber.exceptions.HttpClientError as e:
pprint(e.content)
raise
def delete_volume(self, vid):
self.log.info("Deleting volume with ID {}".format(vid))
volume = self.volume(vid)
if volume['server'] is not None:
self.log.error("Volume with ID {} is attached to server with ID {} ({})".format(vid, volume['server']['id'],
volume['server']['name']))
return False
try:
resp = self._api.query().volumes(vid).delete()
return resp
except slumber.exceptions.HttpClientError as e:
pprint(e.content)
raise
def poweron_server(self, sid):
self.log.info("Powering ON server with ID {}".format(sid))
return self.server_action(sid, 'poweron', target_state='running', wait_for='stopped')
def poweroff_server(self, sid):
self.log.info("Powering OFF server with ID {}".format(sid))
return self.server_action(sid, 'poweroff', target_state='stopped', wait_for='running')
def reboot_server(self, sid):
self.log.info("Rebooting server with ID {}".format(sid))
return self.server_action(sid, 'reboot')
def server_action(self, sid, action, wait_for=None, target_state=None):
if target_state:
server = self.server(sid)
if server['state'] == target_state:
self.log.debug("Server {} is already in state {}".format(sid, server['state']))
return True
if wait_for:
self.wait_for(sid, wait_for)
try:
resp = self._api.query().servers(sid).action.post({'action': action})
return resp
except slumber.exceptions.HttpClientError as e:
pprint(e.content)
raise
def wait_for(self, sid, state):
self.log.debug("Waiting for server {} to be in state {}".format(sid, state))
server = self.server(sid)
while server['state'] != state:
self.log.debug(
"Server {} is in state {} ({}) waiting for {}".format(sid, server['state'], server['state_detail'],
state))
time.sleep(10)
server = self.server(sid)
self.log.debug("Server {} is in state {}".format(sid, server['state']))
def cleanup(self):
self.log.info('Cleaning up instances')
p = ThreadPool(10)
p.map(self.cleanup_instance, [i['id'] for i in self.instances])
def cleanup_instance(self, sid):
self.log.debug('Cleaning up instance {}'.format(sid))
server = self.server(sid)
self.poweroff_server(sid)
self.delete_server(sid)
for volume in server['volumes'].values():
self.delete_volume(volume['id'])
def create_instances(self):
master_id = 1
agent_id = 1
pub_agent_id = 1
self.log.info('Sending master instance creation requests')
while master_id <= self._args.masters:
server = self.create_server('master{}'.format(master_id), self.images['CentOS 7 (beta)']['id'],
self._args.flavor, {}, ['master', self._args.cid])
self.poweron_server(server['id'])
self.instances.append(server)
master_id += 1
self.log.info('Sending agent instance creation requests')
while agent_id <= self._args.agents:
server = self.create_server('agent{}'.format(agent_id), self.images['CentOS 7 (beta)']['id'],
self._args.flavor, {}, ['agent', self._args.cid])
self.poweron_server(server['id'])
self.instances.append(server)
agent_id += 1
self.log.info('Sending public agent instance creation requests')
while pub_agent_id <= self._args.pub_agents:
server = self.create_server('master{}'.format(pub_agent_id), self.images['CentOS 7 (beta)']['id'],
self._args.flavor, {}, ['public_agent', self._args.cid])
self.poweron_server(server['id'])
self.instances.append(server)
pub_agent_id += 1
wait = True
while wait:
time.sleep(10)
wait = False
for idx, server in enumerate(self.instances):
updated_server = self.server(server['id'])
self.instances[idx] = updated_server
if updated_server['public_ip'] is None:
self.log.debug(
'Server {} ({}) is in state {} ({})'.format(updated_server['name'], updated_server['id'], updated_server['state'],
updated_server['state_detail']))
else:
self.log.debug(
'Server {} ({}) is in state {} ({})'.format(updated_server['name'], updated_server['private_ip'],
updated_server['state'],
updated_server['state_detail']))
if updated_server['state'] != 'running':
wait = True
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.DEBUG)
AUTH_TOKEN = os.getenv("AUTH_TOKEN")
SCHEDULE_FILE = os.getenv(
"SCHEDULE_FILE"
) # path to schedule file, for instance: ./schedule_file.json
if not AUTH_TOKEN:
raise ValueError("AUTH_TOKEN not found.")
if not SCHEDULE_FILE:
raise ValueError("SCHEDULE_FILE not found.")
api = ComputeAPI(auth_token=AUTH_TOKEN)
servers = api.query().servers.get()
datetime_format = "%Y-%m-%d %H:%M %z"
# Read schedule file
with open(SCHEDULE_FILE) as json_file:
for instance in json.load(json_file):
logging.info(f"\n")
logging.info(f"{instance['instance_name'].upper()}")
logging.info('-' * 50)
# check is the instance exists in list of servers
instance_id = None
for server in servers['servers']:
if server['name'] == instance['instance_name']:
instance_id = server['id']
import json
import secrets
import requests
from scaleway.apis import ComputeAPI
num_open_jobs = requests.get(
'http://mdma.vis.one/num_open_jobs').json()['num_open_jobs']
api = ComputeAPI(region='ams1', auth_token=secrets.token)
servers = api.query().servers.get()
# find server by name
server_id, server_state = [[x['id'], x['state']]
for x in servers['servers'] if x['name'] == 'mdma-worker'][0]
print(server_id)
print(server_state)
if num_open_jobs > 0 and server_state == 'stopped':
print(api.query().servers(server_id).action.post({'action': 'poweron'}))
if num_open_jobs == 0 and server_state == 'running':
print(api.query().servers(server_id).action.post({'action': 'poweroff'}))