def env_client(self, env):
if not self.url.endswith('/'):
self.url = self.url + '/'
env_url = urljoin(self.url, 'projects/%s' % env)
env_conf_file = CONF_ENV_FILE_PATH % env
if os.path.exists(env_conf_file):
with open(env_conf_file, 'r') as conf_file:
conf = json.loads(conf_file.read())
return gdapi.Client(url=env_url,
access_key=conf['access_key'],
secret_key=conf['secret_key'])
else:
global_client = self.global_client()
api_key = global_client.create_apiKey(accountId=env, name='cli')
access_key = api_key.publicValue
secret_key = api_key.secretValue
with open(env_conf_file, 'w') as conf_file:
conf = {
'url': env_url,
'access_key': access_key,
'secret_key': secret_key
}
conf_file.writelines(json.dumps(conf))
sleep(1)
return gdapi.Client(url=env_url,
access_key=access_key,
secret_key=secret_key)
def process(self):
try:
changed = False
if self.params['create_keys'] is not None and self.params[
'access_key'] is None:
if self.check_mode:
self.exit_json(changed=True)
return
client = gdapi.Client(url=self.params['url'] + 'v2-beta/')
name = 'Ansible Key'
description = 'Created by Ansible Rancher module'
if self.params['create_keys'] != "True":
try:
params = eval(self.params['create_keys'])
if 'name' in params:
name = params['name']
if 'description' in params:
description = params['description']
except:
self.log('exception')
pass
main_key = client.create('apiKey',
name=name,
description=description)
self.params['access_key'] = main_key['publicValue']
self.params['secret_key'] = main_key['secretValue']
self._facts['api_key'] = {
'secret_key': self.params['secret_key'],
'access_key': self.params['access_key']
}
self._account_client = gdapi.Client(
url=self.params['url'] + 'v2-beta/',
access_key=self.params['access_key'],
secret_key=self.params['secret_key'])
self._catalog_client = gdapi.Client(
url=self.params['url'] + 'v1-catalog/',
access_key=self.params['access_key'],
secret_key=self.params['secret_key'])
if self.params['catalogs'] is not None:
changed |= self.check_catalogs(self.params['catalogs'])
if self.params['environments'] is not None:
## not that nice, but no point in updating the envs when cleaning
changed |= self.check_environments(self.params['environments'])
if self.params['clean_hosts']:
self.remove_disconnected_hosts()
if self.params['setup_auth'] is not None:
self.setup_auth(self._account_client,
self.params['setup_auth'])
self.exit_json(changed=changed,
ansible_facts={self.params['var']: self._facts},
output=self._output)
except urllib2.HTTPError as e:
error_message = e.read()
self.fail_json(msg=error_message)
except Exception as e:
self.fail_json(msg=e.message)
def get_single_service_status(self, module_version):
stack_name = self.get_stack_name(module_version)
rancher = gdapi.Client(url=self.RANCHER_URL,
access_key=self.RANCHER_ACCESS_KEY,
secret_key=self.RANCHER_SECRET_KEY)
# lookup environment id (this may become a deployment config option)
slist = rancher.list_environment(name=stack_name)
if len(slist) == 0:
return None
eid = slist[0]['id']
# get service info
entry = rancher.list_service(
environmentId=eid, name=self.get_service_name(module_version))
if len(entry) == 0:
return None
entry = entry[0]
status = {
'module_name': module_version['module_name'],
'release_tags': module_version['release_tags'],
'git_commit_hash': module_version['git_commit_hash'],
'hash': module_version['git_commit_hash'],
'version': module_version['version'],
'url': self.get_service_url(module_version),
'status': entry['state'],
'health': entry['healthState'],
'up': 1 if entry['state'] == 'active' else 0
}
return status
def set_stack_description(self, module_version):
pprint('setting stack description')
rancher = gdapi.Client(url=self.RANCHER_URL,access_key=self.RANCHER_ACCESS_KEY,secret_key=self.RANCHER_SECRET_KEY)
stacks = rancher.list_environment(name=self.get_stack_name(module_version))
pprint(len(stacks))
pprint(module_version)
if (len(stacks) > 0):
exportConfigURL=stacks[0]['actions']['update']
payload = {'description': module_version['git_url'] }
x = requests.put(exportConfigURL, data = json.dumps(payload), auth=(self.RANCHER_ACCESS_KEY,self.RANCHER_SECRET_KEY),verify=False)
def get_rancher_client():
if 'RANCHER_API_URL' not in os.environ:
raise Exception('RANCHER_API_URL env var must be set')
if 'RANCHER_ACCESS_KEY' not in os.environ:
raise Exception('RANCHER_ACCESS_KEY env var must be set')
if 'RANCHER_SECRET_KEY' not in os.environ:
raise Exception('RANCHER_SECRET_KEY env var must be set')
return gdapi.Client(url=os.getenv('RANCHER_API_URL'),
access_key=os.getenv('RANCHER_ACCESS_KEY'),
secret_key=os.getenv('RANCHER_SECRET_KEY'))
def create_compose_files(self, module_version):
# in progress: pull the existing config from rancher and include in new config
# 1) look up service name to get project/environment
# 2) POST {"serviceIds":[]} to /v1/projects/$projid/environments/$envid/?action=exportconfig
# parse dockerComposeConfig and rancherComposeConfig as yml
# construct the service names
service_name = self.get_service_name(module_version)
dns_service_name = self.get_dns_service_name(module_version)
rancher = gdapi.Client(url=self.RANCHER_URL,
access_key=self.RANCHER_ACCESS_KEY,
secret_key=self.RANCHER_SECRET_KEY)
stacks = rancher.list_environment(
name=self.get_stack_name(module_version))
# there should be only one stack, but what if there is more than one?
is_new_stack = False
if (stacks is None or len(stacks) == 0):
is_new_stack = True
# code to fetch existing docker_compose and rancher_compose files
#if len(stacks) > 0:
# exportConfigURL=stacks[0]['actions']['exportconfig']
# payload = {'serviceIds':[]}
# configReq = requests.post(exportConfigURL, data = json.dumps(payload), auth=(self.RANCHER_ACCESS_KEY,self.RANCHER_SECRET_KEY),verify=False)
# export=configReq.json()
# docker_compose = yaml.load(export['dockerComposeConfig'])
# rancher_compose = yaml.load(export['rancherComposeConfig'])
docker_compose = {}
rancher_compose = {}
docker_compose[dns_service_name] = {
"image": "rancher/dns-service",
"links": [service_name + ':' + service_name]
}
docker_compose[service_name] = {
"image": module_version['docker_img_name'],
"labels": {
'us.kbase.module.version':
module_version['version'],
'us.kbase.module.git_commit_hash':
module_version['git_commit_hash']
},
"environment": {
'KBASE_ENDPOINT': self.KBASE_ENDPOINT
}
}
rancher_compose[service_name] = {"scale": 1}
return docker_compose, rancher_compose, is_new_stack
def main():
module = AnsibleModule(
argument_spec = dict(
url = dict(required=False),
access_key = dict(required=False),
secret_key = dict(required=False),
name = dict(required=False),
account = dict(required=False),
publicValue = dict(required=False),
secretValue = dict(required=False),
state = dict(default='present', choices=['present', 'absent'])
),
supports_check_mode = True
)
url = module.params['url']
access_key = module.params['access_key']
secret_key = module.params['secret_key']
name = module.params['name']
account = module.params['account']
publicValue = module.params['publicValue']
secretValue = module.params['secretValue']
state = module.params['state']
try:
client = gdapi.Client(url=url, access_key=access_key, secret_key=secret_key)
apikey = False
for entry in client.list_api_key():
if entry.name == name and entry.publicValue == publicValue:
apikey = entry
break
changed = False
if state == 'present':
if apikey:
changed = True
response = client.update(apikey, {'accountId': account, 'name': name, 'publicValue': publicValue, 'secretValue': secretValue})
module.exit_json(changed=True, key=module.params)
else:
response = client.create_api_key({'accountId': account, 'name': name, 'publicValue': publicValue, 'secretValue': secretValue})
module.exit_json(changed=True, key=module.params)
elif state == 'absent':
if apikey:
apikey.deactivate()
apikey.remove()
module.exit_json(changed=True)
else:
module.exit_json(changed=False)
module.exit_json(changed=False)
except gdapi.ApiError as e:
module.fail_json(msg=str(e.error))
except Exception as e:
module.fail_json(msg=str(e))
def __init__(self, *args, **kwargs):
super(ServiceLoopingSchedule, self).__init__(*args, **kwargs)
self.client = gdapi.Client(url=Config.api_url(),
access_key=Config.access_key(),
secret_key=Config.secret_key())
self.base_param = {
'endTime_null': True,
'limit': 100,
'sort': 'id',
'order': 'desc'
}
def __init__(self, name):
self._access_key = os.environ['RANCHER_ACCESS_KEY']
self._secret_key = os.environ['RANCHER_SECRET_KEY']
self._url = os.environ['RANCHER_URL']
self._client = gdapi.Client(url=self._url,
access_key=self._access_key,
secret_key=self._secret_key)
self.act_serv = actionlib.SimpleActionServer(name + '/control',
RancherCtrlAction,
self.act_serv_execute,
False)
self.act_serv.start()
def process(self):
try:
changed = False
self._account_client = gdapi.Client(
url=self.params['url'] + 'v2-beta/',
access_key=self.params['access_key'],
secret_key=self.params['secret_key'])
self._catalog_client = gdapi.Client(
url=self.params['url'] + 'v1-catalog/',
access_key=self.params['access_key'],
secret_key=self.params['secret_key'])
if 'catalogs' in self.params:
changed |= self.check_catalogs(self.params['catalogs'])
if 'environments' in self.params:
changed |= self.check_environments(self.params['environments'])
self.exit_json(changed=changed,
ansible_facts={'rancher': self._facts},
output=self._output,
catalogs=self._catalogs)
except Exception as e:
self.fail_json(msg=e.message)
def get_client(self, use_catalog_api=False):
self.ensure_one()
scheme = 'https' if self.is_ssl else 'http'
with open(self.secret_path, 'r') as fh:
secret_key = fh.read().strip()
version = self.catalog_version if use_catalog_api else self.version
return gdapi.Client(
url='%s://%s:%s/%s' % (
scheme,
self.host,
self.port,
version,
),
access_key=self.access_key,
secret_key=secret_key,
)
def remount(self, mount_path, projectName=None):
s = self.app.services
log = self.app.log
rancher_url = os.environ['RANCHER_URL']
rancher_access_key = os.environ['RANCHER_ACCESS_KEY']
rancher_secret_key = os.environ['RANCHER_SECRET_KEY']
client = gdapi.Client(url=rancher_url,
access_key=rancher_access_key,
secret_key=rancher_secret_key)
project = None
if (projectName):
project = client.list_project(name=projectName)[0]
else:
project = client.list_project()[0]
for volume in self.get_volumes(mount_path):
if len(client.list_volume(name=volume).data) <= 0:
s.rancher_service.add_volume(volume, project['id'])
log.info('Mounted volume \'' + volume + '\'')
def global_client(self):
return gdapi.Client(url=self.url,
access_key=self.access_key,
secret_key=self.secret_key)
import copy
import os
import time
import gdapi
from japronto import Application
from apscheduler.schedulers.asyncio import AsyncIOScheduler
client = None
if 'CATTLE_ACCESS_KEY' in os.environ and 'CATTLE_SECRET_KEY' in os.environ:
client = gdapi.Client(url=os.environ['CATTLE_URL'].replace('v1', 'v2-beta'),
access_key=os.environ['CATTLE_ACCESS_KEY'],
secret_key=os.environ['CATTLE_SECRET_KEY'])
else:
client = gdapi.Client(url=os.environ['CATTLE_URL'].replace('v1', 'v2-beta'))
STACKS = {}
PROJECTS = None
OLD_CONTAINERS = []
STAY_TIME = float(float(os.getenv('STAY_TIME', 7)) * 86400)
STAY_TIME_STACK = float(float(os.getenv('STAY_TIME_STACK', 1)) * 86400)
CLEANUP_STACKS = os.getenv('CLEANUP_STACKS', '').split(',')
CLEANUP_SERVICE_IN_STACKS = os.getenv('CLEANUP_SERVICE_IN_STACKS', '').split(',')
async def index(request):
return request.Response(json=[])
async def get_project_and_stacks():
global STACKS
def create_compose_files(self, module_version, secure_param_list,
volume_mounts):
# in progress: pull the existing config from rancher and include in new config
# 1) look up service name to get project/environment
# 2) POST {"serviceIds":[]} to /v1/projects/$projid/environments/$envid/?action=exportconfig
# parse dockerComposeConfig and rancherComposeConfig as yml
# construct the service names
service_name = self.get_service_name(module_version)
dns_service_name = self.get_dns_service_name(module_version)
rancher = gdapi.Client(url=self.RANCHER_URL,
access_key=self.RANCHER_ACCESS_KEY,
secret_key=self.RANCHER_SECRET_KEY)
stacks = rancher.list_environment(
name=self.get_stack_name(module_version))
# there should be only one stack, but what if there is more than one?
is_new_stack = False
if (stacks is None or len(stacks) == 0):
is_new_stack = True
# code to fetch existing docker_compose and rancher_compose files
#if len(stacks) > 0:
# exportConfigURL=stacks[0]['actions']['exportconfig']
# payload = {'serviceIds':[]}
# configReq = requests.post(exportConfigURL, data = json.dumps(payload), auth=(self.RANCHER_ACCESS_KEY,self.RANCHER_SECRET_KEY),verify=False)
# export=configReq.json()
# docker_compose = yaml.load(export['dockerComposeConfig'])
# rancher_compose = yaml.load(export['rancherComposeConfig'])
docker_compose = {}
rancher_compose = {}
docker_compose[dns_service_name] = {
"image": "rancher/dns-service",
"links": [service_name + ':' + service_name]
}
environ_map = {
'KBASE_ENDPOINT': self.KBASE_ENDPOINT,
'AUTH_SERVICE_URL': self.AUTH_SERVICE_URL,
'AUTH_SERVICE_URL_ALLOW_INSECURE':
self.AUTH_SERVICE_URL_ALLOW_INSECURE
}
for secure_param in secure_param_list:
param_name = secure_param['param_name']
param_value = secure_param['param_value']
environ_map['KBASE_SECURE_CONFIG_PARAM_' +
param_name] = param_value
docker_compose[service_name] = {
"image": module_version['docker_img_name'],
"labels": {
'us.kbase.dynamicservice':
"True",
'us.kbase.module.version':
module_version['version'],
'us.kbase.module.git_commit_hash':
module_version['git_commit_hash']
},
"environment": environ_map
}
if len(volume_mounts) > 0:
mounts = []
for vol in volume_mounts:
if vol['read_only'] > 0:
mounts.append('%s:%s:ro' %
(vol['host_dir'], vol['container_dir']))
else:
mounts.append('%s:%s' %
(vol['host_dir'], vol['container_dir']))
docker_compose[service_name]['volumes'] = mounts
rancher_compose[service_name] = {"scale": 1}
return docker_compose, rancher_compose, is_new_stack
def get_service_log_web_socket(self, ctx, params):
"""
returns connection info for a websocket connection to get realtime service logs
:param params: instance of type "GetServiceLogParams" (optional
instance_id to get logs for a specific instance. Otherwise logs
from all instances are returned, TODO: add line number
constraints.) -> structure: parameter "service" of type "Service"
(module_name - the name of the service module, case-insensitive
version - specify the service version, which can be either:
(1) full git commit hash of the module version (2) semantic
version or semantic version specification Note: semantic version
lookup will only work for released versions of the module. (3)
release tag, which is one of: dev | beta | release This
information is always fetched from the Catalog, so for more
details on specifying the version, see the Catalog documentation
for the get_module_version method.) -> structure: parameter
"module_name" of String, parameter "version" of String, parameter
"instance_id" of String
:returns: instance of list of type "ServiceLogWebSocket" ->
structure: parameter "instance_id" of String, parameter
"socket_url" of String
"""
# ctx is the context object
# return variables are: sockets
#BEGIN get_service_log_web_socket
service = params['service']
user_id = ctx['user_id']
cc = Catalog(self.CATALOG_URL, token=ctx['token'])
module = cc.get_module_info({'module_name': service['module_name']})
has_access = False
for o in module['owners']:
if o == user_id:
has_access = True
if not has_access:
if cc.is_admin(user_id) == 1:
has_access = True
if not has_access:
raise ValueError(
'Only module owners and catalog admins can view service logs.')
mv = cc.get_module_version({
'module_name': service['module_name'],
'version': service['version']
})
if 'dynamic_service' not in mv:
raise ValueError(
'Specified module is not marked as a dynamic service. (' +
mv['module_name'] + '-' + mv['git_commit_hash'] + ')')
if mv['dynamic_service'] != 1:
raise ValueError(
'Specified module is not marked as a dynamic service. (' +
mv['module_name'] + '-' + mv['git_commit_hash'] + ')')
rancher = gdapi.Client(url=self.RANCHER_URL,
access_key=self.RANCHER_ACCESS_KEY,
secret_key=self.RANCHER_SECRET_KEY)
#service_info = rancher.list_servicess(name=self.get_service_name(mv))
GET_SERVICE_URL = self.RANCHER_URL + '/v1/services?name=' + self.get_service_name(
mv) + '&include=instances'
service_info = requests.get(GET_SERVICE_URL,
auth=(self.RANCHER_ACCESS_KEY,
self.RANCHER_SECRET_KEY),
verify=False).json()
if len(service_info['data']) == 0:
raise ValueError(
'Unable to fetch service information. That service version may not be available.'
)
if len(service_info['data'][0]['instances']) == 0:
raise ValueError(
'The service version specified has no available container instances.'
)
instances = service_info['data'][0]['instances']
#pprint(instances)
match_instance_id = False
if 'instance_id' in params:
match_instance_id = True
sockets = []
for i in instances:
if match_instance_id and params['instance_id'] != i['id']:
continue
LOG_URL = i['actions']['logs']
payload = {'follow': True}
log_ws = requests.post(LOG_URL,
data=json.dumps(payload),
auth=(self.RANCHER_ACCESS_KEY,
self.RANCHER_SECRET_KEY),
verify=False).json()
sockets.append({
'instance_id':
i['id'],
'socket_url':
log_ws['url'] + '?token=' + log_ws['token']
})
#END get_service_log_web_socket
# At some point might do deeper type checking...
if not isinstance(sockets, list):
raise ValueError(
'Method get_service_log_web_socket return value ' +
'sockets is not type list as required.')
# return the results
return [sockets]
def list_service_status(self, ctx, params):
"""
:param params: instance of type "ListServiceStatusParams" (not yet
implemented funcdef pause(Service service) returns (ServiceStatus
status);) -> structure: parameter "is_up" of type "boolean",
parameter "module_names" of list of String
:returns: instance of list of type "ServiceStatus" (module_name -
name of the service module version - semantic version
number of the service module git_commit_hash - git commit hash of
the service module release_tags - list of release tags
currently for this service module (dev/beta/release) url
- the url of the service up - 1 if the service is up,
0 otherwise status - status of the service as reported by
rancher health - health of the service as reported by
Rancher TODO: add something to return: string
last_request_timestamp;) -> structure: parameter "module_name" of
String, parameter "version" of String, parameter "git_commit_hash"
of String, parameter "release_tags" of list of String, parameter
"hash" of String, parameter "url" of String, parameter "up" of
type "boolean", parameter "status" of String, parameter "health"
of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN list_service_status
rancher = gdapi.Client(url=self.RANCHER_URL,
access_key=self.RANCHER_ACCESS_KEY,
secret_key=self.RANCHER_SECRET_KEY)
cc = Catalog(self.CATALOG_URL, token=ctx['token'])
# first create simple module_name lookup based on hash (TODO: in catalog, allow us to only fetch dynamic service modules)
modules = cc.list_basic_module_info({
'include_released': 1,
'include_unreleased': 1
})
module_hash_lookup = {} # hash => module_name
for m in modules:
if 'dynamic_service' not in m or m['dynamic_service'] != 1:
continue
module_hash_lookup[self.get_module_name_hash(
m['module_name'])] = m['module_name']
# next get environment id (could be a config parameter in the future rather than looping over everything)
result = []
# not sure if this is supposed to pass False, or the string 'false'
slists = rancher.list_environment(system='false')
if len(slists) == 0: return [] # I shouldn't return
for slist in slists:
eid = slist['id']
# get service info
entries = rancher.list_service(environmentId=eid)
if len(entries) == 0: continue
for entry in entries:
rs = entry['name'].split('-')
if len(rs) != 2: continue
es = {
'status': entry['state'],
'health': entry['healthState'],
'hash': rs[1]
}
#if es['health'] == 'healthy' and es['status'] == 'active':
if es['status'] == 'active':
es['up'] = 1
else:
es['up'] = 0
try:
mv = cc.get_module_version({
'module_name':
module_hash_lookup[rs[0]],
'version':
rs[1]
})
es['url'] = self.get_service_url(mv)
es['version'] = mv['version']
es['module_name'] = mv['module_name']
es['release_tags'] = mv['release_tags']
es['git_commit_hash'] = mv['git_commit_hash']
except:
# this will occur if the module version is not registered with the catalog, or if the module
# was not marked as a service, or if something was started in Rancher directly and pulled
# from somewhere else, or an old version of the catalog was used to start this service
es['url'] = "https://{0}:{1}/dynserv/{3}.{2}".format(
self.SVC_HOSTNAME, self.NGINX_PORT, rs[0], rs[1])
es['version'] = ''
es['release_tags'] = []
es['git_commit_hash'] = ''
es['module_name'] = '!' + rs[0] + ''
result.append(es)
returnVal = result
#END list_service_status
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method list_service_status return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def connect(self, key, secret):
import gdapi
self._client = gdapi.Client(url=self._project_uri,
access_key=key,
secret_key=secret)
def env(ctx,
list=False,
create=False,
delete=False,
name=None,
description=None,
type='cattle'):
'''Create and interact with environments/projects'''
rancher = gdapi.Client(url=get_root_api_url(ctx),
access_key=ctx.rancher.access_key,
secret_key=ctx.rancher.secret_key)
if list:
environments = rancher.list_project().data
env_names = [environment.name for environment in environments]
print('')
print('Rancher Environments')
print('--------------------')
print(*env_names, sep='\n')
print('')
elif create:
types = {'swarm', 'kubernetes', 'mesos', 'cattle'}
if name is None:
print()
print('Must specify environment name with -n or -name')
print()
return 1
elif type not in types:
print()
print('Option "-t/--type_of_env" must be one of {0}'.format(types))
print()
return 1
environment = {
"name": name,
"description": description,
"swarm": type == 'swarm',
"kubernetes": type == 'kubernetes',
"mesos": type == 'mesos'
}
rancher.create_project(**environment)
print('')
print("Environment '{}' created".format(name))
elif delete:
environments = rancher.list_project().data
for environment in environments:
if environment.name is name:
environment = rancher.by_id_project(environment.id)
rancher.delete(environment)
print('')
print("Environment '{0}' deleted".format(name))
return
print('')
print('No such environment: {0}'.format(name))
