def authorize_and_check(phase, appname):
role = get_role(phase, appname)
# if unauthorized, refresh first
if role == 'unauthorized':
if not sso_refresh(phase):
error('please login first')
exit(1)
role = get_role(phase, appname)
# after refresh, should be authorized
if role == 'unauthorized':
error('still unauthorized after refresh')
exit(1)
elif role in ['wrong status', 'error', 'unknown']:
error('shit happened when getting role')
exit(1)
elif role == 'no role':
error('you have not a role of the app {}'.format(appname))
error('please contact maintainers of the app')
exit(1)
elif role == 'no app':
# no app, let it go
warn('app {} does not exist, passed by.'.format(appname))
return None
else:
# valid role
return role
def push(phase):
"""
Push release and meta images
"""
check_phase(phase)
info("Pushing meta and release images ...")
yml = lain_yaml(ignore_prepare=True)
meta_version = yml.repo_meta_version()
if meta_version is None:
error("please git commit.")
return None
domain = get_domain(phase)
registry = "registry.%s" % domain
phase_meta_tag = docker.gen_image_name(yml.appname, 'meta', meta_version,
registry)
phase_release_tag = docker.gen_image_name(yml.appname, 'release',
meta_version, registry)
meta_code = docker.push(phase_meta_tag)
release_code = docker.push(phase_release_tag)
if meta_code or release_code:
error("Error lain push.")
sys.exit(1)
else:
info("Done lain push.")
info("notifying lain push.")
access_token = SSOAccess.get_token(phase)
auth_header = get_auth_header(access_token)
last_commit_id = fetch_last_commit_id(domain, yml.appname, auth_header)
if last_commit_id is not None:
notify_diffs(domain, yml.appname, last_commit_id, auth_header)
else:
warn("Notified Nothing!")
info("Done notifying lain push.")
def build(push=False, release=False):
"""
Build release and meta images
"""
info("Building meta and release images ...")
validate_only_warning()
yml = lain_yaml()
meta_version = yml.repo_meta_version()
use_prepare = docker.exist(yml.img_names['prepare'])
use_build = release and docker.exist(yml.img_names['build'])
release_suc, release_name = yml.build_release(use_prepare, use_build)
(meta_suc, meta_name) = (False,
'') if not release_suc else yml.build_meta()
if not (release_suc and meta_suc):
sys.exit(1)
if meta_version is None:
warn("please git commit.")
if push:
if meta_version is None:
error("need git commit SHA1.")
return None
tag_release_name = yml.tag_meta_version(release_name)
docker.push(tag_release_name)
tag_meta_name = yml.tag_meta_version(meta_name)
docker.push(tag_meta_name)
info("Done lain build.")
def push(phase):
"""
Push release and meta images
"""
check_phase(phase)
info("Pushing meta and release images ...")
yml = lain_yaml(ignore_prepare=True)
meta_version = yml.repo_meta_version()
if meta_version is None:
error("please git commit.")
return None
domain = get_domain(phase)
registry = "registry.%s" % domain
phase_meta_tag = docker.gen_image_name(
yml.appname, 'meta', meta_version, registry)
phase_release_tag = docker.gen_image_name(
yml.appname, 'release', meta_version, registry)
meta_code = docker.push(phase_meta_tag)
release_code = docker.push(phase_release_tag)
if meta_code or release_code:
error("Error lain push.")
sys.exit(1)
else:
info("Done lain push.")
info("notifying lain push.")
access_token = SSOAccess.get_token(phase)
auth_header = get_auth_header(access_token)
last_commit_id = fetch_last_commit_id(domain, yml.appname, auth_header)
if last_commit_id is not None:
notify_diffs(domain, yml.appname, last_commit_id, auth_header)
else:
warn("Notified Nothing!")
info("Done notifying lain push.")
def print_available_version(version_list):
if len(version_list) == 0:
warn("No available release versions.")
else:
info("Below are the available versions: ")
for version in version_list:
print(version)
def check_phase(phase):
if phase not in PHASE_CHOICES:
error('phase %s not in available phases: %s.' % (phase, PHASE_CHOICES))
warn(
'phase can be added using: lain config save %s domain {{ domain }}'
% phase)
exit(1)
def refresh(phase):
"""
Refresh sso token
"""
check_phase(phase)
refresh_success = sso_refresh(phase)
if refresh_success:
info("Refresh successfully!")
else:
warn('Refresh failed, Please try again!')
def notify_pushs(domain, appname, auth_header):
headers = {"Content-type": "application/json"}
headers.update(auth_header)
url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname)
try:
resp = requests.request("POST", url, headers=headers, timeout=10)
datas = resp.json()
if resp.status_code < 200 or resp.status_code >= 400:
warn('Notify lain push failed with error: %s' % datas['msg'])
except Exception as e:
warn('Notify lain push failed with error: %s' % e)
def logout(phase):
"""
Logout specific phase
"""
check_phase(phase)
domain = get_domain(phase)
logout_success = SSOAccess.clear_token(phase)
if logout_success:
docker.logout('registry.%s' % domain)
info("Logout successfully!")
else:
warn('Logout failed!')
def notify_pushs(domain, appname, auth_header):
headers = {"Content-type": "application/json"}
headers.update(auth_header)
url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname)
try:
resp = requests.request(
"POST", url, headers=headers, timeout=10)
datas = resp.json()
if resp.status_code < 200 or resp.status_code >= 400:
warn('Notify lain push failed with error: %s' %
datas['msg'])
except Exception as e:
warn('Notify lain push failed with error: %s' % e)
def logout(phase):
"""
Logout specific phase
"""
check_phase(phase)
domain = get_domain(phase)
logout_success = SSOAccess.clear_token(phase)
if logout_success:
docker.logout('registry.%s'%domain)
info("Logout successfully!")
else:
warn('Logout failed!')
def undeploy_app(appname, console, auth_header):
delete_url = "http://%s/api/v1/apps/%s/" % (console, appname)
delete_r = requests.delete(delete_url, headers=auth_header)
try:
if delete_r.status_code == 202:
info("delete app %s success." % appname)
info("delete result details: ")
print(delete_r.json()['msg'])
else:
warn("delete app %s fail: %s" % (appname, delete_r.json()['msg']))
except Exception:
error("shit happend: %s" % delete_r.content)
exit(1)
def undeploy_proc(proc, appname, console, auth_header):
proc_url = "http://%s/api/v1/apps/%s/procs/%s/" % (console, appname, proc)
delete_r = requests.delete(proc_url, headers=auth_header)
try:
if delete_r.status_code == 202:
info("delete proc %s success." % proc)
info("delete result details: ")
print(delete_r.json()['msg'])
else:
warn("delete proc %s fail: %s" % (proc, delete_r.json()['msg']))
except Exception:
error("shit happend: %s" % delete_r.content)
exit(1)
def get_auth_code(self, username, password):
try:
usr_msg = {'login': username, 'password': password}
result = requests.post(
self.auth_url,
data=usr_msg,
allow_redirects=False)
code_callback_url = result.headers['Location']
authentication = parse_qs(urlparse(code_callback_url).query)
return True, authentication['code'][0]
except Exception:
warn("Please insure '%s' is accessable." % self.auth_url)
warn("If not, please specify the sso cid and secret when login.")
return False, ''
def validate():
"""
Validate lain.yaml
"""
valid, msg = _validate()
if valid:
info('valid lain.yaml.')
else:
error('invalid lain.yaml.')
warn('error message:')
info(msg)
# TODO : show related doc url
warn('for details of lain.yaml schema please check related docs.')
sys.exit(1)
def _request(cls, method, phase, url, data):
uri = "http://backupctl.%s/%s" % (get_domain(phase), url)
if method.lower() == 'get':
resp = requests.get(uri)
elif method.lower() == 'post':
resp = requests.post(uri, data)
if resp.status_code >= 300:
warn('backupctl return error: %s' % resp.text)
return None
if not resp.text:
return None
try:
return json.loads(resp.text)
except:
warn("not valid json text: %s" % resp.text)
return ""
def fetch_last_commit_id(domain, appname, auth_header):
url = "http://console.%s/api/v1/repos/%s/details/" % (domain, appname)
commitid_length = 40
try:
resp = requests.request("GET", url, headers=auth_header, timeout=10)
datas = resp.json()
if resp.status_code == 404:
warn('%s' % datas['msg'])
return None
elif resp.status_code < 200 or resp.status_code >= 400:
warn('Fetch lastest meta version failed with error: %s' %
datas['msg'])
return None
details = datas['detail']
meta_version = details.get('meta_version', '')
giturl = details.get('giturl', '')
if giturl.strip() == '':
warn('No Giturl Bound')
return None
last_commit_id = meta_version[-commitid_length:]
return last_commit_id
except Exception:
# do nothing, compatible with old console versions
pass
return None
def fetch_last_commit_id(domain, appname, auth_header):
url = "http://console.%s/api/v1/repos/%s/details/" % (domain, appname)
commitid_length = 40
try:
resp = requests.request("GET", url, headers=auth_header, timeout=10)
datas = resp.json()
if resp.status_code == 404:
warn('%s' % datas['msg'])
return None
elif resp.status_code < 200 or resp.status_code >= 400:
warn('Fetch lastest meta version failed with error: %s' %
datas['msg'])
return None
details = datas['detail']
meta_version = details.get('meta_version', '')
giturl = details.get('giturl', '')
if giturl.strip() == '':
warn('No Giturl Bound')
return None
last_commit_id = meta_version[-commitid_length:]
return last_commit_id
except Exception:
# do nothing, compatible with old console versions
pass
return None
def validate_parameters(cpu, memory, numinstances):
if all([cpu is None, memory is None, numinstances is None]):
warn("please input at least one param in cpu/memory/numinstances")
exit(1)
if numinstances is not None:
try:
numinstances = int(numinstances)
except ValueError:
warn('invalid parameter: num_instances (%s) should be integer' %
numinstances)
exit(1)
if numinstances <= 0:
warn('invalid parameter: num_instances (%s) should > 0' %
numinstances)
exit(1)
if cpu is not None:
try:
cpu = int(cpu)
except ValueError:
warn('invalid parameter: cpu (%s) should be integer' % cpu)
exit(1)
if cpu < 0:
warn('invalid parameter: cpu (%s) should >= 0' % cpu)
exit(1)
if memory is not None:
memory = str(memory)
try:
if humanfriendly.parse_size(memory) < 4194304:
error('invalid parameter: memory (%s) should >= 4M' % memory)
exit(1)
except humanfriendly.InvalidSize:
error(
'invalid parameter: memory (%s) humanfriendly.parse_size(memory) failed'
% memory)
exit(1)
return cpu, memory, numinstances
def validate_parameters(cpu, memory, numinstances):
if all([cpu is None, memory is None, numinstances is None]):
warn("please input at least one param in cpu/memory/numinstances")
exit(1)
if numinstances is not None:
try:
numinstances = int(numinstances)
except ValueError:
warn('invalid parameter: num_instances (%s) should be integer'%numinstances)
exit(1)
if numinstances <= 0:
warn('invalid parameter: num_instances (%s) should > 0'%numinstances)
exit(1)
if cpu is not None:
try:
cpu = int(cpu)
except ValueError:
warn('invalid parameter: cpu (%s) should be integer'%cpu)
exit(1)
if cpu < 0:
warn('invalid parameter: cpu (%s) should >= 0'%cpu)
exit(1)
if memory is not None:
memory = str(memory)
try:
if humanfriendly.parse_size(memory) < 4194304:
error('invalid parameter: memory (%s) should >= 4M'%memory)
exit(1)
except humanfriendly.InvalidSize:
error('invalid parameter: memory (%s) humanfriendly.parse_size(memory) failed'%memory)
exit(1)
return cpu, memory, numinstances
def sso_refresh(phase):
sso_access = SSOAccess.new(phase, None, None, None)
refresh_token = sso_access.get_refresh_token(phase)
if not refresh_token:
warn('refresh failed, no refresh token got')
return False
refresh_success, new_access_token, new_refresh_token = sso_access.refresh_auth_token(refresh_token)
if not refresh_success:
warn('refresh failed, refresh_auth_token failed ')
return False
save_token_success = sso_access.save_token(phase, new_access_token, new_refresh_token)
if not save_token_success:
warn('refresh failed, save_token failed')
return False
return True
def sso_login(phase, cid, secret, redirect_uri, username, password):
sso_access = SSOAccess.new(phase, cid, secret, redirect_uri)
get_code_success, code = sso_access.get_auth_code(username, password)
if not get_code_success:
warn('get_auth_code failed, username or password may be wrong.')
return False
get_token_success, access_token, refresh_token = sso_access.get_auth_token(code)
if not get_token_success:
warn('get_auth_token failed, sso client secret may be wrong.')
return False
save_token_success = sso_access.save_token(phase, access_token, refresh_token)
if not save_token_success:
warn('save_token failed')
return False
return True
def notify_diffs(domain, appname, last_commit_id, auth_header):
current_git_commit = git_commit_id()
if current_git_commit == last_commit_id:
warn('Nothing Changed!')
return
unique_authors = git_authors(last_commit_id, 'HEAD')
commits_info = git_commits(last_commit_id, 'HEAD')
headers = {"Content-type": "application/json"}
headers.update(auth_header)
url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname)
body = {'authors': unique_authors, 'commits': commits_info}
try:
resp = requests.request(
"POST", url, headers=headers, json=body, timeout=10)
datas = resp.json()
if resp.status_code < 200 or resp.status_code >= 400:
warn('Notify lain push failed with error: %s' %
datas['msg'])
except Exception as e:
warn('Notify lain push failed with error: %s' % e)
def notify_diffs(domain, appname, last_commit_id, auth_header):
current_git_commit = git_commit_id()
if current_git_commit == last_commit_id:
warn('Nothing Changed!')
return
unique_authors = git_authors(last_commit_id, 'HEAD')
commits_info = git_commits(last_commit_id, 'HEAD')
headers = {"Content-type": "application/json"}
headers.update(auth_header)
url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname)
body = {'authors': unique_authors, 'commits': commits_info}
try:
resp = requests.request("POST",
url,
headers=headers,
json=body,
timeout=10)
datas = resp.json()
if resp.status_code < 200 or resp.status_code >= 400:
warn('Notify lain push failed with error: %s' % datas['msg'])
except Exception as e:
warn('Notify lain push failed with error: %s' % e)
def exit_gracefully(signal, frame):
warn("You pressed Ctrl + C, and I will exit...")
sys.exit(130)
def exit_gracefully(signal, frame):
warn("You pressed Ctrl + C, and I will exit...")
sys.exit(130)
def scale(phase,
proc,
target=None,
cpu=None,
memory=None,
numinstances=None,
output='pretty'):
"""
Scale proc with cpu/memory/num_instances
"""
check_phase(phase)
yml = lain_yaml(ignore_prepare=True)
appname = target if target else yml.appname
authorize_and_check(phase, appname)
domain = get_domain(phase)
console = "console.%s" % domain
url = "http://{}/api/v1/apps/{}/procs/{}/".format(console, appname, proc)
access_token = SSOAccess.get_token(phase)
auth_header = get_auth_header(access_token)
cpu, memory, numinstances = validate_parameters(cpu, memory, numinstances)
proc_status = requests.get(url, headers=auth_header)
if proc_status.status_code == 200:
# proc exists
info(
"Start to scale Proc {} of App {} in Lain Cluster {} with domain {}"
.format(proc, appname, phase, domain))
elif proc_status.status_code == 404:
# proc does not exist
warn(
"Proc {} of App {} does not exists in Lain Cluster {} with domain {}"
.format(proc, appname, phase, domain))
info("Please deploy it first")
exit(1)
else:
error("shit happend: %s" % proc_status.content)
exit(1)
scale_results = {}
payload1 = {}
payload2 = {}
if cpu is not None:
payload1['cpu'] = cpu
if memory is not None:
payload1['memory'] = memory
if len(payload1) > 0:
info("Scaling......")
info(str(payload1))
scale_r = requests.patch(url,
headers=auth_header,
data=json.dumps(payload1),
timeout=120)
scale_results['cpu_or_memory'] = {
'payload': payload1,
'success': scale_r.status_code < 300
}
render_scale_result(scale_r, output)
if numinstances is not None:
payload2['num_instances'] = numinstances
if len(payload2) > 0:
info("Scaling...")
info(str(payload2))
scale_r = requests.patch(url,
headers=auth_header,
data=json.dumps(payload2),
timeout=120)
scale_results['num_instances'] = {
'payload': payload2,
'success': scale_r.status_code < 300
}
render_scale_result(scale_r, output)
info("Outline of scale result: ")
for k, v in scale_results.iteritems():
success = v['success']
output_func = None
if success:
output_func = info
else:
output_func = error
output_func(" scale of {} {}".format(
k, "success" if success else "failed"))
output_func(" params: {}".format(v['payload']))
def check_phase(phase):
if phase not in PHASE_CHOICES:
error('phase %s not in available phases: %s.' % (phase, PHASE_CHOICES))
warn(
'phase can be added using: lain config save %s domain {{ domain }}' % phase)
exit(1)
def get_role(phase, appname):
# 404: {"msg": "user yisun4 does not exist in the app sso-ldap\n", "url": "/api/v1/repos/sso-ldap/maintainers/", "role": null}
# 404: {"msg": "app with appname sso-ldap1 not exist", "url": "/api/v1/repos/", "role": null}
# 401: {"msg": "unauthorized : don't have the access to the operation", "url": "/api/v1/docs/", "app": null}
if not is_console_auth_activated(phase):
return 'noauth-admin'
no_role_pattern = re.compile(r"^user (.+) does not exist in the app (.+)$")
no_app_pattern = re.compile(r"^app with appname (.+) not exist, has not been reposited yet")
domain = get_domain(phase)
console = "console.%s" % domain
url = "http://%s/api/v1/repos/%s/roles/" % (console, appname)
auth_header = get_auth_header(SSOAccess.get_token(phase))
r = requests.get(url, headers=auth_header)
if r.status_code == 401:
return 'unauthorized'
elif r.status_code == 200:
try:
r_json = r.json()
return r_json["role"]["role"]
except Exception as e:
error(e)
warn('DEBUG status: {}'.format(r.status_code))
warn('DEBUG result: {}'.format(r.content))
return 'error'
elif r.status_code == 404:
try:
r_json = r.json()
msg = r_json["msg"]
if no_app_pattern.match(msg):
return 'no app'
elif no_role_pattern.match(msg):
return 'no role'
else:
warn('DEBUG status: {}'.format(r.status_code))
warn('unknown result: {}'.format(r_json))
return 'unknown'
except Exception as e:
error(e)
warn('DEBUG status: {}'.format(r.status_code))
warn('DEBUG result: {}'.format(r.content))
return 'error'
else:
warn('DEBUG status: {}'.format(r.status_code))
warn('DEBUG content: {}'.format(r.content))
return 'wrong status'
def scale(phase, proc, target=None, cpu=None, memory=None, numinstances=None, output='pretty'):
"""
Scale proc with cpu/memory/num_instances
"""
check_phase(phase)
yml = lain_yaml(ignore_prepare=True)
appname = target if target else yml.appname
authorize_and_check(phase, appname)
domain = get_domain(phase)
console = "console.%s" % domain
url = "http://{}/api/v1/apps/{}/procs/{}/".format(
console, appname, proc
)
access_token = SSOAccess.get_token(phase)
auth_header = get_auth_header(access_token)
cpu, memory, numinstances = validate_parameters(cpu, memory, numinstances)
proc_status = requests.get(url, headers=auth_header)
if proc_status.status_code == 200:
# proc exists
info(
"Start to scale Proc {} of App {} in Lain Cluster {} with domain {}".format(
proc, appname, phase, domain
)
)
elif proc_status.status_code == 404:
# proc does not exist
warn(
"Proc {} of App {} does not exists in Lain Cluster {} with domain {}".format(
proc, appname, phase, domain
)
)
info("Please deploy it first")
exit(1)
else:
error("shit happend: %s"%proc_status.content)
exit(1)
scale_results = {}
payload1 = {}
payload2 = {}
if cpu is not None:
payload1['cpu'] = cpu
if memory is not None:
payload1['memory'] = memory
if len(payload1) > 0:
info("Scaling......")
info(str(payload1))
scale_r = requests.patch(url, headers=auth_header,
data=json.dumps(payload1), timeout=120)
scale_results['cpu_or_memory'] = {
'payload': payload1,
'success': scale_r.status_code < 300
}
render_scale_result(scale_r, output)
if numinstances is not None:
payload2['num_instances'] = numinstances
if len(payload2) > 0:
info("Scaling...")
info(str(payload2))
scale_r = requests.patch(url, headers=auth_header,
data=json.dumps(payload2), timeout=120)
scale_results['num_instances'] = {
'payload': payload2,
'success': scale_r.status_code < 300
}
render_scale_result(scale_r, output)
info("Outline of scale result: ")
for k, v in scale_results.iteritems():
success = v['success']
output_func = None
if success:
output_func = info
else:
output_func = error
output_func(" scale of {} {}".format(k, "success" if success else "failed"))
output_func(" params: {}".format(v['payload']))
