def run_backup(): """ git clone/pull + rsync + git commit/push. Since Jenkins integration , this command doesn't up the .deb into reprepro. Jenkins will do this.""" params = request.json remotehost = params['host'] appName = params['app'] branch = 'master' language = 'none' command = 'backup' msg_stdout = '' msg_return = '' msg_git_push = '' ''' Clone/pull from git + Rsync ''' msg_stdout = action.build_from_git(appName, branch, language, command, remotehost) l.info(msg_stdout['Stdout']) ''' Add new changes, commit and push ''' if not msg_stdout['Error']: msg_git_push = action.git_push(appName, branch) concatenate_git_output = '\n\n %s' % msg_git_push msg_stdout['Stdout'] += concatenate_git_output ''' Jenkins will make the new package from source ''' if msg_stdout['Stdout']: msg_return = msg_stdout['Stdout'] else: msg_return = 'Unknown error: please contact with Buanarepo Admin or take a look to Buanarepo-Server logs.' l.error(msg_return) return msg_return
def run_git(): """ If a git repository doesn't exists into buanarepo's build path (ej. /srv/buanarepo-build), This repository will be cloned. Later, a new .deb pkg will be created """ params = request.json appName = params['app'] branch = params['branch'] language = params['language'] command = 'git' msg_return = '' msg_stdout = '' # Clone or pull Git repository msg_stdout = action.build_from_git(appName, branch, language, command, remotehost=None) if msg_stdout['Stdout']: msg_return = msg_stdout['Stdout'] else: msg_return = 'Unknown error: please contact with Buanarepo Admin or take a look to Buanarepo-Server logs.' l.error(msg_return) return msg_return
def listPackage(self): l_packages = [] pkg_file = None basic_auth = (API_USER, USER_PASS) try: request = get(URL_PKG, pkcs12_filename=CERT_P12, pkcs12_password=CERT_P12_PASS,\ verify=CA_CERT, auth=basic_auth) #request = urllib2.Request(URL_PKG) #response = urllib2.urlopen(request, context=ssl._create_unverified_context()) pkg_file = request.text except Exception as e: l.error(e) result_packages = e if not pkg_file: l_packages = result_packages else: result_packages = re.findall('Package.*', pkg_file) for package in result_packages: pkg = package.split() l_packages.append(pkg[1]) return l_packages
def find_ldap_user(attrlist, dn): ''' 从ldap批量取用户信息 :param attrlist: 需要的字段,如 ['sAMAccountName', 'name', 'userAccountControl', 'msExchHomeServerName'] :param dn: DN条件,查询的范围 :return: account_list普通账号的信息 ''' import ldap.controls # 分页查询 page_control = ldap.controls.SimplePagedResultsControl(True, size=1000, cookie='') # 最后生成的用户列表 account_list = [] try: con = con_ldap() while 1: msgid = con.search_ext(dn, ldap.SCOPE_SUBTREE, attrlist=attrlist, serverctrls=[page_control]) a, res_data, b, srv_ctrls = con.result3(msgid) for info in res_data: # TODO :此处可去除这两种条件限制,在后面过滤 # msExchHomeServerName 表明用户具有exchange server邮箱,而不是其他类型的AD账号 if 'userAccountControl' in str(info[1]) and 'Exchange' in str( info[1]): if PY2: # userAccountControl记录AD账号的属性,该属性标志是累积性的,32不需要密码,512是普通账号66048=65536+512,514是禁用账号, if info[1]['userAccountControl'][0] in [ '512', '544', '66048', '66080', '262656', '262688', '328192', '328224' ]: account_list.append(info) elif PY3: if info[1]['userAccountControl'][0] in [ b'512', b'544', b'66048', b'66080', b'262656', b'262688', b'328192', b'328224' ]: account_list.append(info) cookie = srv_ctrls[0].cookie if cookie: page_control.cookie = cookie else: break except Exception as e: logger.error(e) return account_list
def build_from_git(self, appName, branch, language, command, remotehost): ''' call clone_from_git and build package from source cloned ''' msg_stdout_git = '' msg_build_pkg = '' msg_stdout = self.clone_from_git(appName, language) if not msg_stdout['Error']: if command == 'backup': appLoader = AppLoader(MOD_PATH) """ It's like 'from lib.Kafka import Kafka' and 'loadClass = Kafka()' """ try: l.info('Loading %s class for sync...' % appName) loadClass = appLoader.get_instance(appName) except ImportError as e: stderr = str(e) msg_stdout['Stdout'] = 'Error: Application "%s" not found'\ % appName msg_stdout['Error'] = True l.error(stderr) else: try: msg_stdout['Stdout'] = loadClass.backup(remotehost) l.info(msg_stdout['Stdout']) except Exception as e: msg_exception = str(e) l.error(msg_exception) msg_stdout['Stdout'] = msg_exception msg_stdout['Error'] = True else: try: msg_build_pkg = self.build_pkg(appName, command, branch,language) l.info(msg_build_pkg['Stdout']) msg_stdout['Stdout'] = msg_build_pkg['Stdout'] msg_stdout['Error'] = msg_build_pkg['Error'] except Exception as e: msg_stdout['Stdout'] = str(e) msg_stdout['Error'] = True return msg_stdout
def verify_password(request_username, request_password): access = '' if request_username in users: m = md5.new() m.update(request_password) md5_pass = m.hexdigest() if md5_pass == users.get(request_username): l.info('User/Passwd Authentication is OK!') access = True else: l.error( 'Wrong User or Password. Please edit password and try again.') access = False else: l.error('User not found!') return access
def run_mpkg(): """ Create package from source whitout pull/clon the code.""" params = request.json appName = params['app'] branch = params['branch'] language = params['language'] default_command = 'mpkg' msg_return = {} # Make the package msg_return = action.make_pkg(appName, branch, language) if msg_return['Stdout']: msg_return = msg_return['Stdout'] else: msg_return = 'Unknown error: please contact with Buanarepo Admin or take a look to Buanarepo-Server logs.' l.error(msg_return) return msg_return
def useSubprocess(self, cmd): msg_return = {'Stdout': '', 'Error': False} stdout = '' stderr = '' try: p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() except Exception as e: l.error(e) msg_return['Stdout'] = e msg_return['Error'] = True if not stdout: msg_return['Stdout'] = stderr msg_return['Error'] = True else: msg_return['Stdout'] = stdout return msg_return
def git_push(self, appName, branch): ''' Commit the changes ''' appName = str(appName).lower() path_local_repo = '%s/%s' % (BASE_BUILD, appName) remote_url = 'https://%s:[email protected]/%s/' % (TOKEN, COMPANY) remote_repo = remote_url + appName msg_stdout = '' try: repo = Repo(path_local_repo) except InvalidGitRepositoryError as e: msg_stdout = str(e) l.error(msg_stdout) except NoSuchPathError as e: msg_stdout = str(e) l.error(msg_stdout) else: origin = repo.remote(name='origin') """ If in this point is present the next API error "list out of range", the problem is git control file""" try: l.info('Commit the new changes at %s' % path_local_repo) git = repo.git git.add ('.') repo.index.commit('New %s configuration' % appName) origin.push() msg_stdout = 'The %s latest version was pushed!' % appName except Exception as e: msg_stdout = str(e) l.error(msg_stdout) return msg_stdout
def clone_from_git(self, appName, language): appName = str(appName).lower() path_local_repo = '%s/%s' % (BASE_BUILD, appName) remote_url = 'https://%s:[email protected]/%s/'\ % (TOKEN, COMPANY) remote_repo = remote_url + appName msg_stdout = { 'Stdout' : 'Error: %s isn\'t a valid local repository'\ % path_local_repo, 'Error' : True } try: repo = Repo(path_local_repo) msg_stdout['Error'] = False except InvalidGitRepositoryError as e: l.error(msg_stdout['Stdout']) msg_stdout['Error'] = True except NoSuchPathError: try: l.info('Clonning %s repository' % remote_repo) msg_stdout['Stdout'] = Repo.clone_from(remote_repo,\ path_local_repo, depth=1) msg_stdout['Error'] = False except GitCommandError as gitException: if re.findall(r'Repository not found.', gitException.stderr): msg_stdout['Stdout'] = gitException.stderr msg_stdout['Error'] = True l.error(msg_stdout['Stdout']) else: l.info('Repository %s already exists, doing pull...' % path_local_repo) origin = repo.remote(name='origin') """ If in this point is present the next API error "list out of range", the problem is git control file""" try: msg_stdout['Stdout'] = origin.pull() msg_stdout['Error'] = False l.info(msg_stdout['Stdout']) except Exception as e: msg_stdout['Stdout'] = str(e) msg_stdout['Error'] = True l.error(msg_stdout['Stdout']) return msg_stdout
def make_sync(self, appName, branch): msg_return = '' appName = str(appName).lower() cmd_ls = CMD_LS % (DEBS_BASE, appName) search_package = self.useSubprocess(cmd_ls) if not search_package['Error']: package = search_package['Stdout'] for component in COMPONENTS: cmd_remove = CMD_REMOVE % (UBUNTU_BASE, branch, component,\ appName) remove_package = self.useSubprocess(cmd_remove) if not remove_package['Error']: cmd_upload = CMD_UPLOAD % (UBUNTU_BASE, branch,\ component, DEBS_BASE, package) l.info(remove_package['Stdout']) upload_package = self.useSubprocess(cmd_upload) if upload_package['Error']: l.error(upload_package['Stdout']) raise Exception(msg_return) else: msg_return += upload_package['Stdout'] l.info(upload_package['Stdout']) else: msg_return = remove_package['Stdout'] l.error(remove_package['Stdout']) raise Exception(msg_return) else: msg_return = search_package['Stdout'] l.error(search_package['Stdout']) raise Exception(msg_return) return msg_return
def orchestrator(self, host, services, l_msg): RemoteDeployUser = REMOTE_DEPLOY_USER UpdateCommand = UPDATE_CMD InstallCommand = INSTALL_CMD msg_action = [] proc = os.getpid() try: client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy()) client.connect(host, username=RemoteDeployUser, timeout=30.0) upd_cmd = UpdateCommand stdin, stdout, stderr = client.exec_command(upd_cmd) except SSHException as error: cmd_stdout = self.create_msg(host, error[0], 'host') l.error(cmd_stdout) msg_action.append(cmd_stdout) except AuthenticationException as error: cmd_stdout = self.create_msg(host, error[0], 'host') l.error(cmd_stdout) msg_action.append(cmd_stdout) except BadHostKeyException as error: cmd_stdout = self.create_msg(host, error[0], 'host') l.error(cmd_stdout) msg_action.append(cmd_stdout) except Exception as error: cmd_stdout = self.create_msg(host, 'Destination Host Unreachable',\ 'host') l.error(cmd_stdout) msg_action.append(cmd_stdout) else: for service in services: ins_cmd = '%s %s' % (InstallCommand, service) l.info(ins_cmd) try: stdin, stdout, stderr = client.exec_command(ins_cmd) exit_status = stdout.channel.recv_exit_status() if exit_status == 0: cmd_stdout = self.create_msg(service, 'ok', 'service') l.info(cmd_stdout) msg_action.append(cmd_stdout) else: cmd_stdout = self.create_msg(service, 'error', 'service') l.error(cmd_stdout) msg_action.append(cmd_stdout) except SSHException as error: cmd_stdout = self.create_msg(service, error[0], 'service') l.error(cmd_stdout) msg_action.append(cmd_stdout) """When the command finishes executing, the channel will be closed and can’t be reused. You must open a new channel if you wish to execute another command.""" client.close() msg_action = {'Host': host, 'Install': msg_action} l_msg.append(msg_action)
def build_pkg(self, appName, command, branch, language=None): """ Builds packages from local dirs or git repositories. """ multiPackagePath = MULTI_PACKAGE_PATH path_local_repo = '%s/%s' % (BASE_BUILD, appName) local_jenkins_repo = '%s/%s' % (BASE_JENKINS_BUILD, appName) appName = str(appName).lower() msg_return = {'Stdout': '', 'Error': False} msg_exception = '' if command == 'git': base_path = '%s/%s' % (path_local_repo, appName) appDpkgSource = '%s/%s' % (appName, appName) # If multi-package folder exists, then process sub-packages in it: sub_package_path = '%s/%s' % (path_local_repo, multiPackagePath) elif command == 'mpkg': base_path = '%s/%s' % (local_jenkins_repo, appName) appDpkgSource = '%s/%s' % (appName, appName) sub_package_path = '%s/%s' % (local_jenkins_repo, multiPackagePath) elif command == 'backup': base_path = '%s' % (path_local_repo) appDpkgSource = appName l.info('Checks multi-package path: %s' % sub_package_path) if os.path.isdir(sub_package_path): l.info('Running multi-package case...') try: appBaseSource = '%s/%s' % (appName, multiPackagePath) msg_return = self.build_several_pkgs(sub_package_path,\ appBaseSource, command, branch, language) if msg_return['Error']: l.error(msg_return['Stdout']) else: l.info(msg_return['Stdout']) except Exception as msg_exception: l.error(msg_exception) msg_return['Stdout'] = str(msg_exception) msg_return['Error'] = True else: l.info('No multi-package case detected.') l.info('Building %s...' % appName) # Single package case try: control_file = '%s/DEBIAN/control' % base_path appVersion = self.get_app_version(control_file) pkgName = self.get_app_name(control_file) msg_return = self.dpkg(appName, appDpkgSource, appVersion,\ pkgName, command, branch, language) if msg_return['Error']: l.error(msg_return['Stdout']) else: l.info(msg_return['Stdout']) except Exception as msg_exception: l.error(msg_exception) msg_return['Stdout'] = str(msg_exception) msg_return['Error'] = True return msg_return