def post(self, grid_name):
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
args = groupparsers[grid.provider].parse_args()
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_deployment_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
if not groups[grid.provider].select().where(
groups[grid.provider].name == args['name'],
groups[grid.provider].parentgrid ==
grid).exists():
group = groups[grid.provider].create(
parentgrid=grid, name=args['name'])
group = groups[grid.provider].select().where(
groups[grid.provider].name == args['name'],
groups[grid.provider].parentgrid == grid).get()
if group.parentgrid.provider == 'custom':
slaves_args = [args['groupips']]
elif group.parentgrid.provider == 'openstack':
slaves_args = [args['slaves']]
else:
slaves_args = [args['cpus'], args['ram'], args['instance_type']]
group._slaves = self._slave_calculator[grid.provider](
self, *slaves_args)
for key in group._data.keys():
if key != 'id' and key != 'parentgrid' and key != '_slaves':
setattr(group, key, args[key])
group.save()
return ast.literal_eval(str(group)), 200
else:
return 'Group {} already exists'.format(args['name']), 409
def post(self):
args = grid_parser.parse_args()
if not GridEntity.select().where(
GridEntity.name == args['name']).exists():
grid = GridEntity.create(name=args['name'].lower(),
provider=args['provider'],
type=args['type'])
grid.config = configs[grid.provider].create(parentgrid=grid)
grid.deployment = deployments[grid.provider].create(
parentgrid=grid)
return ast.literal_eval(str(grid)), 200
else:
return 'Grid {} already exists'.format(args['name']), 409
def post(self):
args = grid_parser.parse_args()
if not GridEntity.select().where(
GridEntity.name == args['name']).exists():
grid = GridEntity.create(
name=args['name'].lower(),
provider=args['provider'],
type=args['type'])
grid.config = configs[grid.provider].create(
parentgrid=grid)
grid.deployment = deployments[grid.provider].create(
parentgrid=grid)
return ast.literal_eval(str(grid)), 200
else:
return 'Grid {} already exists'.format(args['name']), 409
def __init__(self, grid_name, api_user, api_pass, api_url, **kwargs):
self.grid_name = grid_name
self.api_user = urllib.unquote(api_user)
self.api_pass = urllib.unquote(api_pass)
self.api_url = urllib.unquote(api_url)
self.current_grid = GridEntity.select().where(GridEntity.name == grid_name).get()
self.current_config = (
configs[self.current_grid.provider]
.select()
.where(configs[self.current_grid.provider].parentgrid == self.current_grid)
.get()
)
self.private_key_text = urllib.unquote(self.current_config.sshkeydata)
self.private_key = RSA.importKey(self.private_key_text.strip())
self.public_key_text = self.private_key.publickey().exportKey("OpenSSH")
self.current_groups = []
if not os.path.exists("result/{}/infrastructure".format(grid_name)):
os.makedirs("result/{}/infrastructure".format(grid_name))
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.config = AutoDict()
self.networking = AutoDict()
self.security = AutoDict()
self.terminal = AutoDict()
self.masters = AutoDict()
def _abort_if_deployment_doesnt_exist(self, grid_name):
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
if not deployments[grid.provider].select().where(
deployments[grid.provider].parentgrid == grid).exists():
abort(404,
message="Deployment of grid {} "
"doesn't exist".format(grid_name))
def __init__(self, grid_name, api_user, api_pass, api_url, **kwargs):
self.grid_name = grid_name
self.api_user = urllib.unquote(api_user)
self.api_pass = urllib.unquote(api_pass)
self.api_url = urllib.unquote(api_url)
self.current_grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
self.current_config = configs[
self.current_grid.provider].select().where(
configs[self.current_grid.provider].parentgrid ==
self.current_grid).get()
self.private_key_text = urllib.unquote(self.current_config.sshkeydata)
self.private_key = RSA.importKey(self.private_key_text.strip())
self.public_key_text = self.private_key.publickey().exportKey('OpenSSH')
self.current_groups = []
if not os.path.exists('result/{}/infrastructure'.format(grid_name)):
os.makedirs('result/{}/infrastructure'.format(grid_name))
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.config = AutoDict()
self.networking = AutoDict()
self.security = AutoDict()
self.terminal = AutoDict()
self.masters = AutoDict()
def put(self, grid_name, group_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_deployment_doesnt_exist(grid_name)
self._abort_if_group_doesnt_exist(grid_name, group_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
group = groups[grid.provider].select().where(
groups[grid.provider].name == group_name,
groups[grid.provider].parentgrid == grid).get()
oldgroup = group
args = groupparsers[grid.provider].parse_args()
for key in group._data.keys():
if key != 'id' and key != 'parentgrid' and key != '_slaves':
setattr(group, key, args[key])
if group.parentgrid.provider == 'custom':
slaves_args = [args['groupips']]
elif group.parentgrid.provider == 'openstack':
slaves_args = [args['slaves']]
else:
slaves_args = [args['cpus'], args['ram'], args['instance_type']]
group._slaves = self._slave_calculator[grid.provider](
self, *slaves_args)
group.save()
if args['name'] != oldgroup.name:
deletegroup = groups[grid.provider].select().where(
groups[grid.provider].name == oldgroup.name,
groups[grid.provider].parentgrid == grid).get()
deletegroup.delete_instance()
return ast.literal_eval(str(group)), 200
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
return ast.literal_eval(str(config)), 200
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
return ast.literal_eval(str(config)), 200
def __init__(self, grid_name, *args, **kwargs):
self.grid_name = grid_name
self.current_grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
self.current_config =\
configs[self.current_grid.provider].select().where(
configs[self.current_grid.provider].parentgrid ==
self.current_grid).get()
def get(self, grid_name):
exportgroups = []
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
for group in groups[grid.provider].select():
if group.parentgrid.name == grid_name:
exportgroups.append({'name': group.name, 'role': group.role})
return exportgroups
def __init__(self, grid_name, *args, **kwargs):
self.grid_name = grid_name
self.current_grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
self.current_config =\
configs[self.current_grid.provider].select().where(
configs[self.current_grid.provider].parentgrid ==
self.current_grid).get()
def _abort_if_provision_deployment_doesnt_exist(self, grid_name):
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
if not provision_deployments[grid.provider].select().where(
provision_deployments[grid.provider
].parentgrid == grid).exists():
abort(404, message="Provision Deployment of grid {} "
"doesn't exist".format(grid_name))
def _abort_if_config_doesnt_exist(self, grid_name):
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
if not configs[grid.provider].select().where(
configs[grid.provider].parentgrid ==
grid).exists():
abort(404, message="Config of grid {} doesn't exist".format(
grid_name))
def delete(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
config.delete_instance()
return '', 200
def _abort_if_group_doesnt_exist(self, grid_name, group_name):
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
if not groups[grid.provider].select().where(
groups[grid.provider].name == group_name,
groups[grid.provider].parentgrid ==
grid).exists():
abort(404, message="Group {} doesn't exist".format(group_name))
def delete(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
config.delete_instance()
return '', 200
def _abort_if_infrastructure_deployment_was_not_successful(
self, grid_name):
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
infrastructure_deployment = infrastructure_deployments[
grid.provider].select().where(infrastructure_deployments[
grid.provider].parentgrid == grid).get()
if infrastructure_deployment._status != 'deployed':
abort(404,
message="Infrastructure Deployment of grid need to"
" be successful first".format(grid_name))
def __init__(self, grid_name, **kwargs):
self.grid_name = grid_name
self.kwargs = kwargs
self.current_grid = GridEntity.select().where(GridEntity.name == grid_name).get()
self.current_config = configs[self.current_grid.provider].select().where(configs[self.current_grid.provider].parentgrid == self.current_grid).get()
self.current_groups = []
self.current_roles = []
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.current_roles.append(group.role)
def __init__(self, grid_name, **kwargs):
self.grid_name = grid_name
self.kwargs = kwargs
self.current_grid = GridEntity.select().where(GridEntity.name == grid_name).get()
self.current_config = configs[self.current_grid.provider].select().where(configs[self.current_grid.provider].parentgrid == self.current_grid).get()
self.current_groups = []
self.current_roles = []
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.current_roles.append(group.role)
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_provision_deployment_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
provision_deployment = provision_deployments[
grid.provider].select().where(
provision_deployments[
grid.provider].parentgrid == grid).get()
return ast.literal_eval(str(provision_deployment)), 200
def _abort_if_infrastructure_deployment_was_not_successful(
self, grid_name):
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
infrastructure_deployment = infrastructure_deployments[
grid.provider].select().where(
infrastructure_deployments[
grid.provider].parentgrid == grid).get()
if infrastructure_deployment._status != 'deployed':
abort(404, message="Infrastructure Deployment of grid need to"
" be successful first".format(grid_name))
def put(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
args = configparsers[grid.provider].parse_args()
config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
for key in config._data.keys():
if key != 'id' and key != 'parentgrid':
setattr(config, key, args[key])
config.save()
return ast.literal_eval(str(config)), 200
def put(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
args = configparsers[grid.provider].parse_args()
config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
for key in config._data.keys():
if key != 'id' and key != 'parentgrid':
setattr(config, key, args[key])
config.save()
return ast.literal_eval(str(config)), 200
def __init__(self, grid_name, aws_access_key_id, aws_secret_access_key, **kwargs):
self.aws_access_key_id = urllib.unquote(aws_access_key_id)
self.aws_secret_access_key = urllib.unquote(aws_secret_access_key)
self.grid_name = grid_name
self.kwargs = kwargs
self.current_grid = GridEntity.select().where(GridEntity.name == grid_name).get()
self.current_config = configs[self.current_grid.provider].select().where(configs[self.current_grid.provider].parentgrid == self.current_grid).get()
self.current_groups = []
self.current_roles = []
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.current_roles.append(group.role)
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_deployment_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_was_not_successful(grid_name)
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
infrastructure_deployment = infrastructure_deployments[
grid.provider].select().where(infrastructure_deployments[
grid.provider].parentgrid == grid).get()
with open('gridapi/resources/templates/vpn/vpn.ovpn', 'r') as src:
template = jinja2.Template(src.read())
template_content = template.render(
accessip=infrastructure_deployment._accessip)
return template_content, 200
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_deployment_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_was_not_successful(
grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
infrastructure_deployment = infrastructure_deployments[grid.provider].select().where(
infrastructure_deployments[grid.provider].parentgrid == grid).get()
with open('gridapi/resources/templates/vpn/vpn.ovpn', 'r') as src:
template = jinja2.Template(src.read())
template_content = template.render(accessip=infrastructure_deployment._accessip)
return template_content, 200
def __init__(self, grid_name, credentials, **kwargs):
self.grid_name = grid_name
self.credentials = urllib.unquote(credentials)
self.current_grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
self.current_config = configs[self.current_grid.provider].select(
).where(configs[self.current_grid.provider].parentgrid ==
self.current_grid).get()
self.current_groups = []
if not os.path.exists('result/{}/infrastructure'.format(grid_name)):
os.makedirs('result/{}/infrastructure'.format(grid_name))
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.config = AutoDict()
self.networking = AutoDict()
self.security = AutoDict()
self.storage = AutoDict()
self.terminal = AutoDict()
self.masters = AutoDict()
def __init__(self, grid_name, credentials, **kwargs):
self.grid_name = grid_name
self.credentials = urllib.unquote(credentials)
self.current_grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
self.current_config = configs[
self.current_grid.provider].select().where(
configs[self.current_grid.provider].parentgrid ==
self.current_grid).get()
self.current_groups = []
if not os.path.exists('result/{}/infrastructure'.format(grid_name)):
os.makedirs('result/{}/infrastructure'.format(grid_name))
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.config = AutoDict()
self.networking = AutoDict()
self.security = AutoDict()
self.storage = AutoDict()
self.terminal = AutoDict()
self.masters = AutoDict()
def __init__(self, grid_name, aws_access_key_id, aws_secret_access_key, **kwargs):
self.grid_name = grid_name
self.aws_access_key_id = urllib.unquote(aws_access_key_id)
self.aws_secret_access_key = urllib.unquote(aws_secret_access_key)
self.current_grid = GridEntity.select().where(GridEntity.name == grid_name).get()
self.current_config = (
configs[self.current_grid.provider]
.select()
.where(configs[self.current_grid.provider].parentgrid == self.current_grid)
.get()
)
self.current_groups = []
if not os.path.exists("result/{}/infrastructure".format(grid_name)):
os.makedirs("result/{}/infrastructure".format(grid_name))
for group in groups[self.current_grid.provider].select():
if group.parentgrid.name == grid_name:
self.current_groups.append(group)
self.config = AutoDict()
self.networking = AutoDict()
self.security = AutoDict()
self.terminal = AutoDict()
self.masters = AutoDict()
def delete(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
grid.delete_instance()
return '', 200
def _abort_if_grid_doesnt_exist(self, grid_name):
if not GridEntity.select().where(
GridEntity.name == grid_name).exists():
abort(404, message="Grid {} doesn't exist".format(grid_name))
def put(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_deployment_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_was_not_successful(
grid_name)
self._abort_if_provision_deployment_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
parent_deployment = deployments[grid.provider].select().where(
deployments[grid.provider].parentgrid == grid).get()
infrastructure_deployment = infrastructure_deployments[
grid.provider].select().where(
infrastructure_deployments[
grid.provider].parentgrid == grid).get()
provision_deployment = provision_deployments[
grid.provider].select().where(
provision_deployments[
grid.provider].parentgrid == grid).get()
kwargs = provision_deploymentparsers[grid.provider].parse_args()
cwd = os.getcwd()
if not os.path.exists('result/{}/infrastructure'.format(grid_name)):
os.makedirs('result/{}/infrastructure'.format(grid_name))
if grid.provider != 'custom':
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'w+') as state_file:
state_file.write(infrastructure_deployment._state)
provision_deployment.save()
def do_deploy():
def _run_ansible():
subprocess.check_call(['ansible-playbook', 'masters.yml'],
cwd='result/{}'.format(grid_name))
parent_deployment._status = 'masters_provision_deploying'
provision_deployment._status = 'masters_deploying'
provision_deployment.save()
parent_deployment.save()
provision_generator = provision_generators[grid.type][
grid.provider](grid_name, **kwargs)
provision_generator.generate_all(
grid_name, infrastructure_deployment._accessip)
try:
_run_ansible()
except:
provision_deployment._status = 'masters_deploy_failed'
parent_deployment._status = 'masters_provision_deploy_failed'
else:
provision_deployment._status = 'masters_deployed'
parent_deployment._status = 'masters_provision_finished'
finally:
provision_deployment.save()
self.unlock(parent_deployment)
parent_deployment.save()
os.chdir(cwd)
try:
del os.environ['AWS_ACCESS_KEY_ID']
except:
print('no such env')
try:
del os.environ['AWS_SECRET_ACCESS_KEY']
except:
print('no such env')
shutil.rmtree('result/{}'.format(grid_name))
try:
self.lock(parent_deployment)
parent_deployment.save()
except:
parent_deployment._status = 'lock_failed'
else:
deploy_thread = threading.Thread(
target=do_deploy, args=(), kwargs={})
deploy_thread.start()
return ast.literal_eval(str(provision_deployment)), 200
def get(self):
grids = []
for grid in GridEntity.select():
grids.append({'name': grid.name, 'provider': grid.provider})
return grids
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_was_not_successful(
grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
infrastructure_deployment = infrastructure_deployments[
grid.provider].select().where(
infrastructure_deployments[
grid.provider].parentgrid == grid).get()
def _aws_hosts_export(grid_name):
export = []
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'r') as input_file:
state = json.load(input_file)
for module in state['modules']:
for resource, value in module['resources'].iteritems():
if value['type'] == 'aws_instance' or value['type'] == 'aws_spot_instance_request':
ip = value['primary']['attributes']['private_ip']
export.append(('.'.join(resource.split('.')[1:]), ip))
export.sort(key=lambda(x): x[0])
return export
def _azure_hosts_export(grid_name):
export = []
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'r') as input_file:
state = json.load(input_file)
for module in state['modules']:
for resource, value in module['resources'].iteritems():
if value['type'] == 'azure_instance':
ip = value['primary']['attributes']['vip_address']
export.append(('.'.join(resource.split('.')[1:]), ip))
export.sort(key=lambda(x): x[0])
return export
def _gcs_hosts_export(grid_name):
export = []
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'r') as input_file:
state = json.load(input_file)
for module in state['modules']:
for resource, value in module['resources'].iteritems():
if value['type'] == 'google_compute_instance':
ip = value['primary']['attributes']['network_interface.0.address']
export.append(('.'.join(resource.split('.')[1:]), ip))
export.sort(key=lambda(x): x[0])
return export
def _openstack_hosts_export(grid_name):
export = []
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'r') as input_file:
state = json.load(input_file)
for module in state['modules']:
for resource, value in module['resources'].iteritems():
if value['type'] == 'openstack_compute_instance_v2':
ip = value['primary']['attributes']['network.0.fixed_ip_v4']
export.append(('.'.join(resource.split('.')[1:]), ip))
export.sort(key=lambda(x): x[0])
return export
def _custom_hosts_export(grid_name):
export = []
grid_config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
export.append(('terminal', grid_config.terminalips.split(',')[1]))
for ip in grid_config.mastersips.split(','):
export.append(('master', ip))
for group in groups[grid.provider].select():
if group.parentgrid.name == grid_name:
for ip in group.groupips.split(','):
export.append(('group_{}_host'.format(group.name), ip))
return export
hosts_export = {
'aws': _aws_hosts_export,
'azure': _azure_hosts_export,
'gcs': _gcs_hosts_export,
'openstack': _openstack_hosts_export,
'custom': _custom_hosts_export
}
if not os.path.exists('result/{}/infrastructure'.format(grid_name)):
os.makedirs('result/{}/infrastructure'.format(grid_name))
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'w+') as state_file:
state_file.write(infrastructure_deployment._state)
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'r') as state_file:
export = hosts_export[grid.provider](grid_name)
shutil.rmtree('result/{}'.format(grid_name))
return ast.literal_eval(str(export)), 200
def get(self):
grids = []
for grid in GridEntity.select():
grids.append({'name': grid.name,
'provider': grid.provider})
return grids
def delete(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_infrastructure_deployment_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
parent_deployment = deployments[grid.provider].select().where(
deployments[grid.provider].parentgrid == grid).get()
infrastructure_deployment = infrastructure_deployments[
grid.provider].select().where(
infrastructure_deployments[
grid.provider].parentgrid == grid).get()
args = infrastructure_deploymentparsers[grid.provider].parse_args()
cwd = os.getcwd()
def do_destroy():
parent_deployment._status = 'infrastructure_destroying'
infrastructure_deployment._status = 'destroying'
infrastructure_deployment.save()
parent_deployment.save()
deployment_generator = infrastructure_generators[
grid.provider](grid_name, **args)
deployment_generator.generate_all()
@retry(stop_max_attempt_number=5, wait_fixed=10000)
def _run_terraform_destroy():
subprocess.check_call([
'terraform',
'destroy',
'-force',
'-state=result/{}/infrastructure/terraform.'
'tfstate'.format(grid_name),
'result/{}/infrastructure'.format(grid_name)])
try:
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'w+') as state_file:
state_file.write(infrastructure_deployment._state)
_run_terraform_destroy()
with open('result/{}/infrastructure/terraform.tfstate'.format(
grid_name), 'r') as state_file:
infrastructure_deployment._state = state_file.read()
parent_deployment._status = 'destroyed'
infrastructure_deployment._status = 'destroyed'
except:
parent_deployment._status = 'destroy_failed'
infrastructure_deployment._status = 'destroy_failed'
finally:
self.unlock(parent_deployment)
parent_deployment.save()
os.chdir(cwd)
try:
del os.environ['AWS_ACCESS_KEY_ID']
except:
print('no such env')
try:
del os.environ['AWS_SECRET_ACCESS_KEY']
except:
print('no such env')
shutil.rmtree('result/{}'.format(grid_name))
try:
self.lock(parent_deployment)
parent_deployment.save()
except:
parent_deployment._status = 'lock_failed'
else:
if grid.provider != 'custom':
destroy_thread = threading.Thread(
target=do_destroy, args=(), kwargs={})
destroy_thread.start()
infrastructure_deployment.delete_instance()
return '', 200
def delete(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
grid.delete_instance()
return '', 200
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
grid = GridEntity.select().where(GridEntity.name == grid_name).get()
return ast.literal_eval(str(grid)), 200
def put(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
self._abort_if_config_doesnt_exist(grid_name)
self._abort_if_deployment_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
parent_deployment = deployments[grid.provider].select().where(
deployments[grid.provider].parentgrid == grid).get()
if not infrastructure_deployments[grid.provider].select().where(
infrastructure_deployments[grid.provider
].parentgrid == grid).exists():
infrastructure_deployment = infrastructure_deployments[
grid.provider].create(parentgrid=grid)
infrastructure_deployment = infrastructure_deployments[
grid.provider].select().where(
infrastructure_deployments[
grid.provider].parentgrid == grid).get()
args = infrastructure_deploymentparsers[grid.provider].parse_args()
cwd = os.getcwd()
if not os.path.exists('result/{}/infrastructure'.format(grid_name)):
os.makedirs('result/{}/infrastructure'.format(grid_name))
for key in infrastructure_deployment._data.keys():
if key != 'id' and key != 'parentgrid' and key !=\
'_lock' and key != '_status' and key !=\
'_accessip' and key != '_state':
setattr(infrastructure_deployment, key, urllib.unquote(args[key]))
infrastructure_deployment.save()
def do_deploy():
def _aws_get_access_ip(grid_name):
terminal_ip = 'dummy'
if os.path.isfile(
'result/{}/infrastructure/terraform.'
'tfstate'.format(grid_name)) and os.access(
'result/{}/infrastructure/terraform.tfstate'.format(
grid_name), os.R_OK):
with open('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name), 'r') as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module['resources'].\
iteritems():
if resource == 'aws_eip.terminal':
terminal_ip = value['primary'][
'attributes']['public_ip']
if terminal_ip == 'dummy':
raise Exception('No terminal ip detected')
else:
return terminal_ip
def _azure_get_access_ip(grid_name):
if os.path.isfile(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name)) and os.access(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name), os.R_OK):
with open(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name), 'r') as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module[
'resources'].iteritems():
if resource == 'azure_instance.terminal':
return value['primary']['attributes'][
'vip_address']
def _gcs_get_access_ip(grid_name):
if os.path.isfile(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name)) and os.access(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name), os.R_OK):
with open(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name), 'r') as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module[
'resources'].iteritems():
if resource == 'google_compute_instance.{}-terminal'.format(grid_name):
return value['primary']['attributes'][
'network_interface.0.access_config.0.assigned_nat_ip']
def _openstack_get_access_ip(grid_name):
if os.path.isfile(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name)) and os.access(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name), os.R_OK):
with open(
'result/{}/infrastructure/terraform.tfstate'.
format(grid_name), 'r') as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module['resources'].iteritems():
if resource == 'openstack_compute_instance_v2.{}-terminal'.format(grid_name):
return value['primary']['attributes']['floating_ip']
def _custom_get_access_ip(grid_name):
grid_config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
terminal_ips = grid_config.terminalips.split(',')
return terminal_ips[0]
get_access_ip = {
'aws': _aws_get_access_ip,
'azure': _azure_get_access_ip,
'gcs': _gcs_get_access_ip,
'openstack': _openstack_get_access_ip,
'custom': _custom_get_access_ip
}
@retry(stop_max_attempt_number=30, wait_fixed=5000)
def _aws_check_host(host, grid_name):
try:
subprocess.check_call([
'ssh', '-F', 'result/{}/ssh_config'.format(
grid_name),
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-o', 'PasswordAuthentication=no',
'-o', 'ConnectTimeout=10',
'{}'.format(host), 'exit'])
print('{} is online'.format(host))
except:
print('{} is offline'.format(host))
raise Exception('host is offline')
@retry(stop_max_attempt_number=30, wait_fixed=5000)
def _azure_check_host(host):
try:
subprocess.check_call([
'tcping', '-q', '-t', '1', '{}'.format(
host), '22'])
except:
raise Exception('host is offline')
@retry(stop_max_attempt_number=30, wait_fixed=5000)
def _gcs_check_host(host):
try:
subprocess.check_call([
'ssh', '-F', 'result/{}/ssh_config'.format(
grid_name),
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-o', 'PasswordAuthentication=no',
'-o', 'ConnectTimeout=10',
'{}'.format(host), 'exit'])
print('{} is online'.format(host))
except:
print('{} is offline'.format(host))
raise Exception('host is offline')
@retry(stop_max_attempt_number=30, wait_fixed=5000)
def _openstack_check_host(host):
try:
subprocess.check_call([
'ssh', '-F', 'result/{}/ssh_config'.format(
grid_name),
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-o', 'PasswordAuthentication=no',
'-o', 'ConnectTimeout=10',
'{}'.format(host), 'exit'])
print('{} is online'.format(host))
except:
print('{} is offline'.format(host))
raise Exception('host is offline')
@retry(stop_max_attempt_number=30, wait_fixed=5000)
def _custom_check_host(host, grid_name):
try:
subprocess.check_call([
'ssh', '-F', 'result/{}/ssh_config'.format(
grid_name),
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-o', 'PasswordAuthentication=no',
'-o', 'ConnectTimeout=10',
'{}'.format(host), 'exit'])
print('{} is online'.format(host))
except:
print('{} is offline'.format(host))
raise Exception('host is offline')
check_host = {
'aws': _aws_check_host,
'azure': _azure_check_host,
'gcs': _gcs_check_host,
'openstack': _openstack_check_host,
'custom': _custom_check_host
}
def _aws_check_hosts_online(grid_name):
if os.path.isfile(
'result/{}/infrastructure/terraform.'
'tfstate'.format(grid_name)) and os.access(
'result/{}/infrastructure/terraform.'
'tfstate'.format(grid_name), os.R_OK):
with open('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name), 'r') as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module['resources'].\
iteritems():
if value['type'] == 'aws_instance':
ip = value['primary']['attributes'][
'private_ip']
check_host[grid.provider](
ip, grid_name)
def _azure_check_hosts_online(grid_name):
if os.path.isfile('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name)) and os.access(
'result/{}/infrastructure/terraform.tfstate'.format(
grid_name), os.R_OK):
with open('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name), "r") as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module['resources'].\
iteritems():
if value['type'] == 'azure_instance':
ip = value['primary']['attributes'][
'vip_address']
check_host[grid.provider](ip)
def _gcs_check_hosts_online(grid_name):
if os.path.isfile('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name)) and os.access(
'result/{}/infrastructure/terraform.tfstate'.format(
grid_name), os.R_OK):
with open('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name), "r") as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module['resources'].\
iteritems():
if value['type'] == 'google_compute_instance':
ip = value['primary']['attributes'][
'network_interface.0.address']
check_host[grid.provider](ip)
def _openstack_check_hosts_online(grid_name):
if os.path.isfile('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name)) and os.access(
'result/{}/infrastructure/terraform.tfstate'.format(
grid_name), os.R_OK):
with open('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name), "r") as json_file:
json_data = json.load(json_file)
for module in json_data['modules']:
for resource, value in module['resources'].\
iteritems():
if value['type'] == 'openstack_compute_instance_v2':
ip = value['primary']['attributes'][
'network.0.fixed_ip_v4']
check_host[grid.provider](ip)
def _custom_check_hosts_online(grid_name):
all_ips = []
grid_config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
all_ips.append(grid_config.terminalips.split(',')[1])
all_ips.extend(grid_config.mastersips.split(','))
for group in groups[grid.provider].select():
if group.parentgrid.name == grid_name:
all_ips.extend(group.groupips.split(','))
for ip in all_ips:
check_host[grid.provider](ip, grid_name)
check_hosts_online = {
'aws': _aws_check_hosts_online,
'azure': _azure_check_hosts_online,
'gcs': _gcs_check_hosts_online,
'openstack': _openstack_check_hosts_online,
'custom': _custom_check_hosts_online
}
def generate_ssh_config(access_ip):
import jinja2
grid_config = configs[grid.provider].select().where(
configs[grid.provider].parentgrid == grid).get()
ssh_user = grid_config.ssh_user
os.system('cp -a -f gridapi/resources/templates/infra'
'structure/* result/{}'.format(grid_name))
path = 'result/{}/ssh_config'.format(grid_name)
with open(path, 'r') as src:
template = jinja2.Template(src.read())
template_content = template.render(access_ip=access_ip, user=ssh_user)
with open(path, 'w') as dst:
dst.write(template_content)
def _run_terraform_deploy():
command = [
'terraform',
'apply',
'-parallelism={}'.format(args['parallelism']),
'-state=result/{}/infrastructure/terraform.'
'tfstate'.format(grid_name),
'result/{}/infrastructure'.format(grid_name)]
subprocess.check_call(command)
parent_deployment._status = 'infrastructure_deploying'
infrastructure_deployment._status = 'deploying'
infrastructure_deployment.save()
parent_deployment.save()
infrastructure_generator = infrastructure_generators[
grid.provider](grid_name, **args)
infrastructure_generator.generate_all()
try:
if grid.provider != 'custom':
with open('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name), 'w+') as state_file:
state_file.write(infrastructure_deployment._state)
_run_terraform_deploy()
with open('result/{}/infrastructure/terraform.'
'tfstate'.format(
grid_name), 'r') as state_file:
infrastructure_deployment._state = state_file.read()
infrastructure_deployment.save()
infrastructure_deployment._accessip = get_access_ip[
grid.provider](grid_name)
infrastructure_deployment.save()
generate_ssh_config(infrastructure_deployment._accessip)
check_hosts_online[grid.provider](grid_name)
except:
infrastructure_deployment._status = 'deploy_failed'
parent_deployment._status = 'infrastructure_deploy_failed'
else:
infrastructure_deployment._status = 'deployed'
parent_deployment._status = 'infrastructure_deployed'
finally:
if grid.provider != 'custom':
with open('result/{}/infrastructure/terraform.tfstate'.format(grid_name), 'r') as state_file:
infrastructure_deployment._state = state_file.read()
infrastructure_deployment.save()
self.unlock(parent_deployment)
parent_deployment.save()
os.chdir(cwd)
try:
del os.environ['AWS_ACCESS_KEY_ID']
except:
print('no such env')
try:
del os.environ['AWS_SECRET_ACCESS_KEY']
except:
print('no such env')
shutil.rmtree('result/{}'.format(grid_name))
try:
self.lock(parent_deployment)
parent_deployment.save()
except:
parent_deployment._status = 'lock_failed'
else:
deploy_thread = threading.Thread(
target=do_deploy, args=(), kwargs={})
deploy_thread.start()
return ast.literal_eval(str(infrastructure_deployment)), 200
def get(self, grid_name):
self._abort_if_grid_doesnt_exist(grid_name)
grid = GridEntity.select().where(
GridEntity.name == grid_name).get()
return ast.literal_eval(str(grid)), 200