def authorize_and_check(phase, appname): role = get_role(phase, appname) # if unauthorized, refresh first if role == 'unauthorized': if not sso_refresh(phase): error('please login first') exit(1) role = get_role(phase, appname) # after refresh, should be authorized if role == 'unauthorized': error('still unauthorized after refresh') exit(1) elif role in ['wrong status', 'error', 'unknown']: error('shit happened when getting role') exit(1) elif role == 'no role': error('you have not a role of the app {}'.format(appname)) error('please contact maintainers of the app') exit(1) elif role == 'no app': # no app, let it go warn('app {} does not exist, passed by.'.format(appname)) return None else: # valid role return role
def push(phase): """ Push release and meta images """ check_phase(phase) info("Pushing meta and release images ...") yml = lain_yaml(ignore_prepare=True) meta_version = yml.repo_meta_version() if meta_version is None: error("please git commit.") return None domain = get_domain(phase) registry = "registry.%s" % domain phase_meta_tag = docker.gen_image_name(yml.appname, 'meta', meta_version, registry) phase_release_tag = docker.gen_image_name(yml.appname, 'release', meta_version, registry) meta_code = docker.push(phase_meta_tag) release_code = docker.push(phase_release_tag) if meta_code or release_code: error("Error lain push.") sys.exit(1) else: info("Done lain push.") info("notifying lain push.") access_token = SSOAccess.get_token(phase) auth_header = get_auth_header(access_token) last_commit_id = fetch_last_commit_id(domain, yml.appname, auth_header) if last_commit_id is not None: notify_diffs(domain, yml.appname, last_commit_id, auth_header) else: warn("Notified Nothing!") info("Done notifying lain push.")
def build(push=False, release=False): """ Build release and meta images """ info("Building meta and release images ...") validate_only_warning() yml = lain_yaml() meta_version = yml.repo_meta_version() use_prepare = docker.exist(yml.img_names['prepare']) use_build = release and docker.exist(yml.img_names['build']) release_suc, release_name = yml.build_release(use_prepare, use_build) (meta_suc, meta_name) = (False, '') if not release_suc else yml.build_meta() if not (release_suc and meta_suc): sys.exit(1) if meta_version is None: warn("please git commit.") if push: if meta_version is None: error("need git commit SHA1.") return None tag_release_name = yml.tag_meta_version(release_name) docker.push(tag_release_name) tag_meta_name = yml.tag_meta_version(meta_name) docker.push(tag_meta_name) info("Done lain build.")
def push(phase): """ Push release and meta images """ check_phase(phase) info("Pushing meta and release images ...") yml = lain_yaml(ignore_prepare=True) meta_version = yml.repo_meta_version() if meta_version is None: error("please git commit.") return None domain = get_domain(phase) registry = "registry.%s" % domain phase_meta_tag = docker.gen_image_name( yml.appname, 'meta', meta_version, registry) phase_release_tag = docker.gen_image_name( yml.appname, 'release', meta_version, registry) meta_code = docker.push(phase_meta_tag) release_code = docker.push(phase_release_tag) if meta_code or release_code: error("Error lain push.") sys.exit(1) else: info("Done lain push.") info("notifying lain push.") access_token = SSOAccess.get_token(phase) auth_header = get_auth_header(access_token) last_commit_id = fetch_last_commit_id(domain, yml.appname, auth_header) if last_commit_id is not None: notify_diffs(domain, yml.appname, last_commit_id, auth_header) else: warn("Notified Nothing!") info("Done notifying lain push.")
def print_available_version(version_list): if len(version_list) == 0: warn("No available release versions.") else: info("Below are the available versions: ") for version in version_list: print(version)
def check_phase(phase): if phase not in PHASE_CHOICES: error('phase %s not in available phases: %s.' % (phase, PHASE_CHOICES)) warn( 'phase can be added using: lain config save %s domain {{ domain }}' % phase) exit(1)
def refresh(phase): """ Refresh sso token """ check_phase(phase) refresh_success = sso_refresh(phase) if refresh_success: info("Refresh successfully!") else: warn('Refresh failed, Please try again!')
def notify_pushs(domain, appname, auth_header): headers = {"Content-type": "application/json"} headers.update(auth_header) url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname) try: resp = requests.request("POST", url, headers=headers, timeout=10) datas = resp.json() if resp.status_code < 200 or resp.status_code >= 400: warn('Notify lain push failed with error: %s' % datas['msg']) except Exception as e: warn('Notify lain push failed with error: %s' % e)
def logout(phase): """ Logout specific phase """ check_phase(phase) domain = get_domain(phase) logout_success = SSOAccess.clear_token(phase) if logout_success: docker.logout('registry.%s' % domain) info("Logout successfully!") else: warn('Logout failed!')
def notify_pushs(domain, appname, auth_header): headers = {"Content-type": "application/json"} headers.update(auth_header) url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname) try: resp = requests.request( "POST", url, headers=headers, timeout=10) datas = resp.json() if resp.status_code < 200 or resp.status_code >= 400: warn('Notify lain push failed with error: %s' % datas['msg']) except Exception as e: warn('Notify lain push failed with error: %s' % e)
def logout(phase): """ Logout specific phase """ check_phase(phase) domain = get_domain(phase) logout_success = SSOAccess.clear_token(phase) if logout_success: docker.logout('registry.%s'%domain) info("Logout successfully!") else: warn('Logout failed!')
def undeploy_app(appname, console, auth_header): delete_url = "http://%s/api/v1/apps/%s/" % (console, appname) delete_r = requests.delete(delete_url, headers=auth_header) try: if delete_r.status_code == 202: info("delete app %s success." % appname) info("delete result details: ") print(delete_r.json()['msg']) else: warn("delete app %s fail: %s" % (appname, delete_r.json()['msg'])) except Exception: error("shit happend: %s" % delete_r.content) exit(1)
def undeploy_proc(proc, appname, console, auth_header): proc_url = "http://%s/api/v1/apps/%s/procs/%s/" % (console, appname, proc) delete_r = requests.delete(proc_url, headers=auth_header) try: if delete_r.status_code == 202: info("delete proc %s success." % proc) info("delete result details: ") print(delete_r.json()['msg']) else: warn("delete proc %s fail: %s" % (proc, delete_r.json()['msg'])) except Exception: error("shit happend: %s" % delete_r.content) exit(1)
def get_auth_code(self, username, password): try: usr_msg = {'login': username, 'password': password} result = requests.post( self.auth_url, data=usr_msg, allow_redirects=False) code_callback_url = result.headers['Location'] authentication = parse_qs(urlparse(code_callback_url).query) return True, authentication['code'][0] except Exception: warn("Please insure '%s' is accessable." % self.auth_url) warn("If not, please specify the sso cid and secret when login.") return False, ''
def validate(): """ Validate lain.yaml """ valid, msg = _validate() if valid: info('valid lain.yaml.') else: error('invalid lain.yaml.') warn('error message:') info(msg) # TODO : show related doc url warn('for details of lain.yaml schema please check related docs.') sys.exit(1)
def _request(cls, method, phase, url, data): uri = "http://backupctl.%s/%s" % (get_domain(phase), url) if method.lower() == 'get': resp = requests.get(uri) elif method.lower() == 'post': resp = requests.post(uri, data) if resp.status_code >= 300: warn('backupctl return error: %s' % resp.text) return None if not resp.text: return None try: return json.loads(resp.text) except: warn("not valid json text: %s" % resp.text) return ""
def fetch_last_commit_id(domain, appname, auth_header): url = "http://console.%s/api/v1/repos/%s/details/" % (domain, appname) commitid_length = 40 try: resp = requests.request("GET", url, headers=auth_header, timeout=10) datas = resp.json() if resp.status_code == 404: warn('%s' % datas['msg']) return None elif resp.status_code < 200 or resp.status_code >= 400: warn('Fetch lastest meta version failed with error: %s' % datas['msg']) return None details = datas['detail'] meta_version = details.get('meta_version', '') giturl = details.get('giturl', '') if giturl.strip() == '': warn('No Giturl Bound') return None last_commit_id = meta_version[-commitid_length:] return last_commit_id except Exception: # do nothing, compatible with old console versions pass return None
def validate_parameters(cpu, memory, numinstances): if all([cpu is None, memory is None, numinstances is None]): warn("please input at least one param in cpu/memory/numinstances") exit(1) if numinstances is not None: try: numinstances = int(numinstances) except ValueError: warn('invalid parameter: num_instances (%s) should be integer' % numinstances) exit(1) if numinstances <= 0: warn('invalid parameter: num_instances (%s) should > 0' % numinstances) exit(1) if cpu is not None: try: cpu = int(cpu) except ValueError: warn('invalid parameter: cpu (%s) should be integer' % cpu) exit(1) if cpu < 0: warn('invalid parameter: cpu (%s) should >= 0' % cpu) exit(1) if memory is not None: memory = str(memory) try: if humanfriendly.parse_size(memory) < 4194304: error('invalid parameter: memory (%s) should >= 4M' % memory) exit(1) except humanfriendly.InvalidSize: error( 'invalid parameter: memory (%s) humanfriendly.parse_size(memory) failed' % memory) exit(1) return cpu, memory, numinstances
def validate_parameters(cpu, memory, numinstances): if all([cpu is None, memory is None, numinstances is None]): warn("please input at least one param in cpu/memory/numinstances") exit(1) if numinstances is not None: try: numinstances = int(numinstances) except ValueError: warn('invalid parameter: num_instances (%s) should be integer'%numinstances) exit(1) if numinstances <= 0: warn('invalid parameter: num_instances (%s) should > 0'%numinstances) exit(1) if cpu is not None: try: cpu = int(cpu) except ValueError: warn('invalid parameter: cpu (%s) should be integer'%cpu) exit(1) if cpu < 0: warn('invalid parameter: cpu (%s) should >= 0'%cpu) exit(1) if memory is not None: memory = str(memory) try: if humanfriendly.parse_size(memory) < 4194304: error('invalid parameter: memory (%s) should >= 4M'%memory) exit(1) except humanfriendly.InvalidSize: error('invalid parameter: memory (%s) humanfriendly.parse_size(memory) failed'%memory) exit(1) return cpu, memory, numinstances
def sso_refresh(phase): sso_access = SSOAccess.new(phase, None, None, None) refresh_token = sso_access.get_refresh_token(phase) if not refresh_token: warn('refresh failed, no refresh token got') return False refresh_success, new_access_token, new_refresh_token = sso_access.refresh_auth_token(refresh_token) if not refresh_success: warn('refresh failed, refresh_auth_token failed ') return False save_token_success = sso_access.save_token(phase, new_access_token, new_refresh_token) if not save_token_success: warn('refresh failed, save_token failed') return False return True
def sso_login(phase, cid, secret, redirect_uri, username, password): sso_access = SSOAccess.new(phase, cid, secret, redirect_uri) get_code_success, code = sso_access.get_auth_code(username, password) if not get_code_success: warn('get_auth_code failed, username or password may be wrong.') return False get_token_success, access_token, refresh_token = sso_access.get_auth_token(code) if not get_token_success: warn('get_auth_token failed, sso client secret may be wrong.') return False save_token_success = sso_access.save_token(phase, access_token, refresh_token) if not save_token_success: warn('save_token failed') return False return True
def notify_diffs(domain, appname, last_commit_id, auth_header): current_git_commit = git_commit_id() if current_git_commit == last_commit_id: warn('Nothing Changed!') return unique_authors = git_authors(last_commit_id, 'HEAD') commits_info = git_commits(last_commit_id, 'HEAD') headers = {"Content-type": "application/json"} headers.update(auth_header) url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname) body = {'authors': unique_authors, 'commits': commits_info} try: resp = requests.request( "POST", url, headers=headers, json=body, timeout=10) datas = resp.json() if resp.status_code < 200 or resp.status_code >= 400: warn('Notify lain push failed with error: %s' % datas['msg']) except Exception as e: warn('Notify lain push failed with error: %s' % e)
def notify_diffs(domain, appname, last_commit_id, auth_header): current_git_commit = git_commit_id() if current_git_commit == last_commit_id: warn('Nothing Changed!') return unique_authors = git_authors(last_commit_id, 'HEAD') commits_info = git_commits(last_commit_id, 'HEAD') headers = {"Content-type": "application/json"} headers.update(auth_header) url = "http://console.%s/api/v1/repos/%s/push/" % (domain, appname) body = {'authors': unique_authors, 'commits': commits_info} try: resp = requests.request("POST", url, headers=headers, json=body, timeout=10) datas = resp.json() if resp.status_code < 200 or resp.status_code >= 400: warn('Notify lain push failed with error: %s' % datas['msg']) except Exception as e: warn('Notify lain push failed with error: %s' % e)
def exit_gracefully(signal, frame): warn("You pressed Ctrl + C, and I will exit...") sys.exit(130)
def scale(phase, proc, target=None, cpu=None, memory=None, numinstances=None, output='pretty'): """ Scale proc with cpu/memory/num_instances """ check_phase(phase) yml = lain_yaml(ignore_prepare=True) appname = target if target else yml.appname authorize_and_check(phase, appname) domain = get_domain(phase) console = "console.%s" % domain url = "http://{}/api/v1/apps/{}/procs/{}/".format(console, appname, proc) access_token = SSOAccess.get_token(phase) auth_header = get_auth_header(access_token) cpu, memory, numinstances = validate_parameters(cpu, memory, numinstances) proc_status = requests.get(url, headers=auth_header) if proc_status.status_code == 200: # proc exists info( "Start to scale Proc {} of App {} in Lain Cluster {} with domain {}" .format(proc, appname, phase, domain)) elif proc_status.status_code == 404: # proc does not exist warn( "Proc {} of App {} does not exists in Lain Cluster {} with domain {}" .format(proc, appname, phase, domain)) info("Please deploy it first") exit(1) else: error("shit happend: %s" % proc_status.content) exit(1) scale_results = {} payload1 = {} payload2 = {} if cpu is not None: payload1['cpu'] = cpu if memory is not None: payload1['memory'] = memory if len(payload1) > 0: info("Scaling......") info(str(payload1)) scale_r = requests.patch(url, headers=auth_header, data=json.dumps(payload1), timeout=120) scale_results['cpu_or_memory'] = { 'payload': payload1, 'success': scale_r.status_code < 300 } render_scale_result(scale_r, output) if numinstances is not None: payload2['num_instances'] = numinstances if len(payload2) > 0: info("Scaling...") info(str(payload2)) scale_r = requests.patch(url, headers=auth_header, data=json.dumps(payload2), timeout=120) scale_results['num_instances'] = { 'payload': payload2, 'success': scale_r.status_code < 300 } render_scale_result(scale_r, output) info("Outline of scale result: ") for k, v in scale_results.iteritems(): success = v['success'] output_func = None if success: output_func = info else: output_func = error output_func(" scale of {} {}".format( k, "success" if success else "failed")) output_func(" params: {}".format(v['payload']))
def get_role(phase, appname): # 404: {"msg": "user yisun4 does not exist in the app sso-ldap\n", "url": "/api/v1/repos/sso-ldap/maintainers/", "role": null} # 404: {"msg": "app with appname sso-ldap1 not exist", "url": "/api/v1/repos/", "role": null} # 401: {"msg": "unauthorized : don't have the access to the operation", "url": "/api/v1/docs/", "app": null} if not is_console_auth_activated(phase): return 'noauth-admin' no_role_pattern = re.compile(r"^user (.+) does not exist in the app (.+)$") no_app_pattern = re.compile(r"^app with appname (.+) not exist, has not been reposited yet") domain = get_domain(phase) console = "console.%s" % domain url = "http://%s/api/v1/repos/%s/roles/" % (console, appname) auth_header = get_auth_header(SSOAccess.get_token(phase)) r = requests.get(url, headers=auth_header) if r.status_code == 401: return 'unauthorized' elif r.status_code == 200: try: r_json = r.json() return r_json["role"]["role"] except Exception as e: error(e) warn('DEBUG status: {}'.format(r.status_code)) warn('DEBUG result: {}'.format(r.content)) return 'error' elif r.status_code == 404: try: r_json = r.json() msg = r_json["msg"] if no_app_pattern.match(msg): return 'no app' elif no_role_pattern.match(msg): return 'no role' else: warn('DEBUG status: {}'.format(r.status_code)) warn('unknown result: {}'.format(r_json)) return 'unknown' except Exception as e: error(e) warn('DEBUG status: {}'.format(r.status_code)) warn('DEBUG result: {}'.format(r.content)) return 'error' else: warn('DEBUG status: {}'.format(r.status_code)) warn('DEBUG content: {}'.format(r.content)) return 'wrong status'
def scale(phase, proc, target=None, cpu=None, memory=None, numinstances=None, output='pretty'): """ Scale proc with cpu/memory/num_instances """ check_phase(phase) yml = lain_yaml(ignore_prepare=True) appname = target if target else yml.appname authorize_and_check(phase, appname) domain = get_domain(phase) console = "console.%s" % domain url = "http://{}/api/v1/apps/{}/procs/{}/".format( console, appname, proc ) access_token = SSOAccess.get_token(phase) auth_header = get_auth_header(access_token) cpu, memory, numinstances = validate_parameters(cpu, memory, numinstances) proc_status = requests.get(url, headers=auth_header) if proc_status.status_code == 200: # proc exists info( "Start to scale Proc {} of App {} in Lain Cluster {} with domain {}".format( proc, appname, phase, domain ) ) elif proc_status.status_code == 404: # proc does not exist warn( "Proc {} of App {} does not exists in Lain Cluster {} with domain {}".format( proc, appname, phase, domain ) ) info("Please deploy it first") exit(1) else: error("shit happend: %s"%proc_status.content) exit(1) scale_results = {} payload1 = {} payload2 = {} if cpu is not None: payload1['cpu'] = cpu if memory is not None: payload1['memory'] = memory if len(payload1) > 0: info("Scaling......") info(str(payload1)) scale_r = requests.patch(url, headers=auth_header, data=json.dumps(payload1), timeout=120) scale_results['cpu_or_memory'] = { 'payload': payload1, 'success': scale_r.status_code < 300 } render_scale_result(scale_r, output) if numinstances is not None: payload2['num_instances'] = numinstances if len(payload2) > 0: info("Scaling...") info(str(payload2)) scale_r = requests.patch(url, headers=auth_header, data=json.dumps(payload2), timeout=120) scale_results['num_instances'] = { 'payload': payload2, 'success': scale_r.status_code < 300 } render_scale_result(scale_r, output) info("Outline of scale result: ") for k, v in scale_results.iteritems(): success = v['success'] output_func = None if success: output_func = info else: output_func = error output_func(" scale of {} {}".format(k, "success" if success else "failed")) output_func(" params: {}".format(v['payload']))