def test_exec_cmd(self): # test 1 cmd1 = ['echo', '"test_exec_cmd"'] output, err = exec_cmd(' '.join(cmd1)) self.assertTrue("test_exec_cmd\n" == output.decode("utf-8")) # test 2 cmd1 = ['not_exists_cmd', '"test_exec_cmd"'] output, err = exec_cmd(' '.join(cmd1)) self.assertEqual('', output.decode("utf-8")) self.assertIn('command not found', err.decode("utf-8"))
def checkout_file(self, filename): """ checkout 文件 :param filename: :return: """ cmd = [ 'cd', self.project_path, '&&', self.git_path, 'checkout', filename ] self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd))) exec_cmd(' '.join(cmd))
def file_exists_in_repo(self, filename="sonar-project.properties"): """ 检测文件是否在git仓库中 :param filename: 要检测的文件 :return: """ result = False self.logger.debug( "[GitOperator] Detects whether the '{0}' file is in the git repository..." .format(filename)) output, _ = exec_cmd(' '.join([ 'cd', self.project_path, '&&', self.git_path, 'ls-files', filename, "--error-unmatch;" ])) if output and filename == output.strip().decode("utf-8"): result = True self.logger.info( "[GitOperator] [+] Found '{0}' files in the gitlab project.". format(filename)) else: self.logger.warning( "[GitOperator] [-] '{0}' file not found.".format(filename)) return result
def sync_code(self): """ :return: """ self._logger.info( "[ScanProject] Start syncing project code into the scan directory..." ) if self._project_type == 'online': # online self._git.checkout_branch_sync_code(self._project_branch, self._force_sync_code) self._branch_commit = self._git.get_branch_last_commit_id() self._logger.info( '[ScanProject] current branch commit:{0}, branch name:{1}'. format(self._branch_commit, self._project_branch)) cmd = " -R {0}/* {1}".format(self._project_path, self._paths_project_scan) self._logger.debug("[ScanProject] cp{0}".format(cmd)) self.cp.exec_match(cmd=cmd) elif self._project_type == 'offline' and self._project_local_path: cmd = " -R {0}/* {1}".format(self._project_local_path, self._paths_project_scan) self._logger.debug("[ScanProject] cp{0}".format(cmd)) self.cp.exec_match(cmd=cmd) else: # offline self.__init_distributed() if self._project_storage_type in self.storage: if self._project_storage_type == 'local': # TODO pass elif self._project_storage_type == 'ftp': # FTP make_dir(self._project_path) local_file = os.path.join(self._project_path, self._project_file_origin_name) remote_file = urlparse(self._project_ssh).path if not os.path.isfile(local_file): self.storage['ftp'].download_file( local_file, remote_file) unzip_cmd = [ '/usr/bin/unzip', '-n', local_file, '-d', self._paths_project_scan ] self._logger.debug( "[ScanProject] Start unzipping the file:[{0}]".format( ' '.join(unzip_cmd))) output, err = exec_cmd(' '.join(unzip_cmd)) self.storage['ftp'].close() if err: raise Exception(err) else: msg = 'Does not support "{0}" storage mode.'.format( self._project_storage_type) raise DistributedDoesNotSupportStorage(msg) self._logger.info( "[ScanProject] Synchronization project code completion.")
def __start_blacklist(self, project): """ :param project: :return: """ try: if not os.path.isfile(self._sonar_scanner_path): msg = "File path for 'sonar_scanner' not found, sonar_scanner_path: {0}".format( self._sonar_scanner_path) raise SonarScannerFailureException(msg) self.sonarqube = SonarAPIHandler( token=self._api_token, logger=project.logger, sonar_server=self._api_domain, http_timeout_retry=self._http_timeout_retry, http_failed_retry=self._http_failed_retry, http_timeout=self._http_timeout, ) self.__make_sonar_config(project) # 开始执行 sonar-scanner 命令 exec_cmd(' '.join([ 'cd', project.scan_path, '&&', self._sonar_scanner_path, '1>>{0} 2>&1 &'.format(project.log_file) ])) if self._grep.exec_match(cmd='"EXECUTION SUCCESS" {0}'.format( project.log_file), match_str='EXECUTION SUCCESS'): self.__sync_issues(project) else: reason = self.__get_failed_reason(project) project.update_scan_message(title='EXECUTION FAILURE!', reason=reason, level=2) raise SonarScannerFailureException( '[SonarScanner] EXECUTION FAILURE! {0}'.format(reason)) except (CodeBaseException, gevent.exceptions.LoopExit) as ex: project.logger.error(ex) raise ex
def get_all_local_branch(self): """ 获取本地的所有分支 :return: """ cmd = ['cd', self.project_path, '&&', self.git_path, 'branch'] self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd))) result, _ = exec_cmd(' '.join(cmd)) if result: result = result.decode("utf-8").split("\n") self.local_branch_list = [r.strip() for r in result if r]
def __kill_sonar_scanner_pid(self, project): """ :param project: :return: """ try: project.logger.info( '[SonarScanner] Get the pid number of the sonar-scanner ...') get_sonar_scanner_cmd = 'ps -ef | grep "cd ' + project.scan_path + ' \&\& ' + self._sonar_scanner_path + '"|grep -v grep|awk \'{print($2)}\'' project.logger.debug( '[SonarScanner] Query pid command: [{0}]'.format( get_sonar_scanner_cmd)) pid, _ = exec_cmd(get_sonar_scanner_cmd) pid = parse_int_or_str(pid) or '' project.logger.debug('[SonarScanner] Get PID: [{0}]'.format(pid)) if pid: sub_pid_cmd = 'ps -ef | grep "Dproject.home=' + project.scan_path + '"|grep -v grep|awk \'{print($2)}\'' sub_pid, _ = exec_cmd(sub_pid_cmd) sub_pid = strip(sub_pid) if '\n' in sub_pid.decode("utf-8"): sub_pid = [ parse_int_or_str(_p) for _p in sub_pid.decode("utf-8").split("\n") if _p ] else: sub_pid = parse_int_or_str(sub_pid) or '' kill_sonar_scanner_cmd = 'kill -9 {0} {1}'.format(pid, sub_pid) status, _ = exec_cmd(kill_sonar_scanner_cmd) if _: project.logger.warning( '[SonarScanner] The termination of the sonar-scanner ' 'process failed with the command: [{0}]'.format( kill_sonar_scanner_cmd)) else: project.logger.info( '[SonarScanner] Process [{0},{1}] is terminated.'. format(pid, sub_pid)) except: pass
def get_branch_last_commit_id(self): """ 得到当前分支的最后一次 commit id :return: """ cmd = [ 'cd', self.project_path, '&&', self.git_path, 'log', '--pretty=oneline', '|', 'awk', '\'{print($1)}\'', '|', 'head -n 1' ] self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd))) result, _ = exec_cmd(' '.join(cmd)) return result.strip().decode("utf-8")
def get_current_branch(self): """ 获得当前程序的分支 :return: """ cmd = [ 'cd', self.project_path, '&&', self.git_path, 'symbolic-ref', '--short', '-q', 'HEAD' ] result, _ = exec_cmd(' '.join(cmd)) self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd))) if result: self.current_branch = result.strip().decode("utf-8") return self.current_branch
def get_commit_author(self, git_file_path): """ 获得git文件中的最后一次提交作者 :param git_file_path: 要检测的文件 :return: """ cmd = [ 'cd', self.project_path, '&&', self.git_path, 'log', '-n', '1', git_file_path, '|', 'sed', '-n', '\'2,1p\'' ] self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd))) result, _ = exec_cmd(' '.join(cmd)) if result: result = result.strip().decode("utf-8").replace('Author:', '') return result
def exec_match(self, cmd, match_str=None): """ :param cmd: :param match_str: :return: """ result = False os.chdir("/tmp") if isinstance(cmd, list): cmd_str = '{0} {1}'.format(self._path, ' '.join(cmd)) else: cmd_str = '{0} {1}'.format(self._path, cmd) output, _ = exec_cmd(cmd_str) if match_str and match_str in output.decode('utf-8'): result = True return result
def start_monitor(self): """ 开始发送心跳监控 :return: """ if conf.agent.monitor_url: logger.info( 'Send registration information to the SeeCode server, url: "{0}".' .format(conf.agent.monitor_url)) # 节点注册 registration_data = { 'hostname': get_hostname(), 'ipv4': '127.0.0.1', 'role': 'client', 'client_version': self.version, 'action': 'node', } if conf.server.domain: domain = urlparse(conf.server.domain).netloc registration_data['ipv4'] = get_localhost_ip(domain=domain) else: registration_data['ipv4'] = get_localhost_ip() self.request.post_data(conf.agent.monitor_url, registration_data) # 服务心跳监控 if conf.general.services: heartbeat_data = { 'hostname': get_hostname(), 'ipv4': registration_data['ipv4'], 'action': 'heartbeat', 'items': [] } for item in conf.general.services: try: if item['keyword'] and item['role'] == 'client': output, err = exec_cmd( 'ps -ef | grep "{0}" |grep -v grep|wc -l'. format(item['keyword'])) if err: logger.error(err) else: result = parse_int(output.strip()) if result == 0: status = 2 else: status = 1 heartbeat_data['items'].append({ 'key': item['key'], 'status': status, }) except Exception as ex: logger.error(ex) if heartbeat_data['items']: services_url = '{0}{1}node/services/'.format( conf.server.domain, conf.server.api_uri) self.request.post_data(services_url, heartbeat_data) else: logger.critical( '"monitor_url" parameter is empty, agent monitoring is not possible.' )
def checkout_branch_sync_code(self, branch_name, sync_code=False): """ 切换分支同步代码 :param branch_name: :param sync_code: :return: """ # 初始化并克隆代码 git_config_path = os.path.join(self.project_path, '.git') if not os.path.isdir(git_config_path): self.logger.info( '[GitOperator] git clone project to "{0}" ...'.format( self.project_path)) cmd = [ self.git_path, 'clone', '--depth 1', self.git_url, self.project_path ] self.logger.info( "[GitOperator] Start executing commands: '{0}'".format( ' '.join(cmd))) _, err = exec_cmd(' '.join(cmd)) if err and 'fatal: Could not read from remote repository.' in err.decode( 'utf-8'): raise GitDoesNotPermissionException( '克隆项目代码失败,当前没有权限。{0}'.format(err)) cmd = [ 'cd', self.project_path, '&&', self.git_path, 'remote', 'add', 'upstream', self.git_url, '&&', self.git_path, 'fetch', '--all', '-p', ] self.logger.info( "[GitOperator] Start executing commands: '{0}'".format( ' '.join(cmd))) exec_cmd(' '.join(cmd)) # 解决多个分支切换导致的文件冲突问题 cmd = [ 'cd', self.project_path, '&&', self.git_path, 'reset', '.', '&&', self.git_path, 'checkout', '.', '&&', self.git_path, 'stash' ] self.logger.debug( "[GitOperator] Start executing commands: '{0}'".format( ' '.join(cmd))) exec_cmd(' '.join(cmd)) self.get_all_local_branch() if branch_name in self.local_branch_list: cmd = [ 'cd', self.project_path, '&&', self.git_path, 'checkout', branch_name ] self.logger.debug( "[GitOperator] Start executing commands: '{0}'".format( ' '.join(cmd))) exec_cmd(' '.join(cmd)) if sync_code: # fetch && pull self.logger.info("[GitOperator] Force update of local code...") cmd = [ 'cd', self.project_path, '&&', self.git_path, 'fetch', '--all', '-p', '&&', self.git_path, 'pull' ] self.logger.debug( "[GitOperator] Start executing commands: '{0}'".format( ' '.join(cmd))) exec_cmd(' '.join(cmd)) # 验证扫描分支是否一致 current_branch = self.get_current_branch() if current_branch.strip() != branch_name.strip(): # 判断本地是否存在分支 self.logger.info( '[GitOperator] Start switching branches, current branch: [{0}], target branch: [{1}].' .format(current_branch.strip(), branch_name.strip())) cmd = [ 'cd', self.project_path, '&&', self.git_path, 'checkout', '-b', branch_name, 'remotes/upstream/{0}'.format(branch_name) ] self.logger.debug( "[GitOperator] Start executing commands: '{0}'".format( ' '.join(cmd))) _last_output, _ = exec_cmd(' '.join(cmd)) current_branch = self.get_current_branch() if current_branch != branch_name.strip(): raise CheckoutBranchException('切换分支失败! {0}'.format(_)) self.logger.info('[GitOperator] Code synchronization completed.') return True