def test_exec_cmd(self):
# test 1
cmd1 = ['echo', '"test_exec_cmd"']
output, err = exec_cmd(' '.join(cmd1))
self.assertTrue("test_exec_cmd\n" == output.decode("utf-8"))
# test 2
cmd1 = ['not_exists_cmd', '"test_exec_cmd"']
output, err = exec_cmd(' '.join(cmd1))
self.assertEqual('', output.decode("utf-8"))
self.assertIn('command not found', err.decode("utf-8"))
def checkout_file(self, filename):
"""
checkout 文件
:param filename:
:return:
"""
cmd = [
'cd', self.project_path, '&&', self.git_path, 'checkout', filename
]
self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd)))
exec_cmd(' '.join(cmd))
def file_exists_in_repo(self, filename="sonar-project.properties"):
"""
检测文件是否在git仓库中
:param filename: 要检测的文件
:return:
"""
result = False
self.logger.debug(
"[GitOperator] Detects whether the '{0}' file is in the git repository..."
.format(filename))
output, _ = exec_cmd(' '.join([
'cd', self.project_path, '&&', self.git_path, 'ls-files', filename,
"--error-unmatch;"
]))
if output and filename == output.strip().decode("utf-8"):
result = True
self.logger.info(
"[GitOperator] [+] Found '{0}' files in the gitlab project.".
format(filename))
else:
self.logger.warning(
"[GitOperator] [-] '{0}' file not found.".format(filename))
return result
def sync_code(self):
"""
:return:
"""
self._logger.info(
"[ScanProject] Start syncing project code into the scan directory..."
)
if self._project_type == 'online': # online
self._git.checkout_branch_sync_code(self._project_branch,
self._force_sync_code)
self._branch_commit = self._git.get_branch_last_commit_id()
self._logger.info(
'[ScanProject] current branch commit:{0}, branch name:{1}'.
format(self._branch_commit, self._project_branch))
cmd = " -R {0}/* {1}".format(self._project_path,
self._paths_project_scan)
self._logger.debug("[ScanProject] cp{0}".format(cmd))
self.cp.exec_match(cmd=cmd)
elif self._project_type == 'offline' and self._project_local_path:
cmd = " -R {0}/* {1}".format(self._project_local_path,
self._paths_project_scan)
self._logger.debug("[ScanProject] cp{0}".format(cmd))
self.cp.exec_match(cmd=cmd)
else: # offline
self.__init_distributed()
if self._project_storage_type in self.storage:
if self._project_storage_type == 'local': # TODO
pass
elif self._project_storage_type == 'ftp': # FTP
make_dir(self._project_path)
local_file = os.path.join(self._project_path,
self._project_file_origin_name)
remote_file = urlparse(self._project_ssh).path
if not os.path.isfile(local_file):
self.storage['ftp'].download_file(
local_file, remote_file)
unzip_cmd = [
'/usr/bin/unzip', '-n', local_file, '-d',
self._paths_project_scan
]
self._logger.debug(
"[ScanProject] Start unzipping the file:[{0}]".format(
' '.join(unzip_cmd)))
output, err = exec_cmd(' '.join(unzip_cmd))
self.storage['ftp'].close()
if err:
raise Exception(err)
else:
msg = 'Does not support "{0}" storage mode.'.format(
self._project_storage_type)
raise DistributedDoesNotSupportStorage(msg)
self._logger.info(
"[ScanProject] Synchronization project code completion.")
def __start_blacklist(self, project):
"""
:param project:
:return:
"""
try:
if not os.path.isfile(self._sonar_scanner_path):
msg = "File path for 'sonar_scanner' not found, sonar_scanner_path: {0}".format(
self._sonar_scanner_path)
raise SonarScannerFailureException(msg)
self.sonarqube = SonarAPIHandler(
token=self._api_token,
logger=project.logger,
sonar_server=self._api_domain,
http_timeout_retry=self._http_timeout_retry,
http_failed_retry=self._http_failed_retry,
http_timeout=self._http_timeout,
)
self.__make_sonar_config(project)
# 开始执行 sonar-scanner 命令
exec_cmd(' '.join([
'cd', project.scan_path, '&&', self._sonar_scanner_path,
'1>>{0} 2>&1 &'.format(project.log_file)
]))
if self._grep.exec_match(cmd='"EXECUTION SUCCESS" {0}'.format(
project.log_file),
match_str='EXECUTION SUCCESS'):
self.__sync_issues(project)
else:
reason = self.__get_failed_reason(project)
project.update_scan_message(title='EXECUTION FAILURE!',
reason=reason,
level=2)
raise SonarScannerFailureException(
'[SonarScanner] EXECUTION FAILURE! {0}'.format(reason))
except (CodeBaseException, gevent.exceptions.LoopExit) as ex:
project.logger.error(ex)
raise ex
def get_all_local_branch(self):
"""
获取本地的所有分支
:return:
"""
cmd = ['cd', self.project_path, '&&', self.git_path, 'branch']
self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd)))
result, _ = exec_cmd(' '.join(cmd))
if result:
result = result.decode("utf-8").split("\n")
self.local_branch_list = [r.strip() for r in result if r]
def __kill_sonar_scanner_pid(self, project):
"""
:param project:
:return:
"""
try:
project.logger.info(
'[SonarScanner] Get the pid number of the sonar-scanner ...')
get_sonar_scanner_cmd = 'ps -ef | grep "cd ' + project.scan_path + ' \&\& ' + self._sonar_scanner_path + '"|grep -v grep|awk \'{print($2)}\''
project.logger.debug(
'[SonarScanner] Query pid command: [{0}]'.format(
get_sonar_scanner_cmd))
pid, _ = exec_cmd(get_sonar_scanner_cmd)
pid = parse_int_or_str(pid) or ''
project.logger.debug('[SonarScanner] Get PID: [{0}]'.format(pid))
if pid:
sub_pid_cmd = 'ps -ef | grep "Dproject.home=' + project.scan_path + '"|grep -v grep|awk \'{print($2)}\''
sub_pid, _ = exec_cmd(sub_pid_cmd)
sub_pid = strip(sub_pid)
if '\n' in sub_pid.decode("utf-8"):
sub_pid = [
parse_int_or_str(_p)
for _p in sub_pid.decode("utf-8").split("\n") if _p
]
else:
sub_pid = parse_int_or_str(sub_pid) or ''
kill_sonar_scanner_cmd = 'kill -9 {0} {1}'.format(pid, sub_pid)
status, _ = exec_cmd(kill_sonar_scanner_cmd)
if _:
project.logger.warning(
'[SonarScanner] The termination of the sonar-scanner '
'process failed with the command: [{0}]'.format(
kill_sonar_scanner_cmd))
else:
project.logger.info(
'[SonarScanner] Process [{0},{1}] is terminated.'.
format(pid, sub_pid))
except:
pass
def get_branch_last_commit_id(self):
"""
得到当前分支的最后一次 commit id
:return:
"""
cmd = [
'cd', self.project_path, '&&', self.git_path, 'log',
'--pretty=oneline', '|', 'awk', '\'{print($1)}\'', '|', 'head -n 1'
]
self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd)))
result, _ = exec_cmd(' '.join(cmd))
return result.strip().decode("utf-8")
def get_current_branch(self):
"""
获得当前程序的分支
:return:
"""
cmd = [
'cd', self.project_path, '&&', self.git_path, 'symbolic-ref',
'--short', '-q', 'HEAD'
]
result, _ = exec_cmd(' '.join(cmd))
self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd)))
if result:
self.current_branch = result.strip().decode("utf-8")
return self.current_branch
def get_commit_author(self, git_file_path):
"""
获得git文件中的最后一次提交作者
:param git_file_path: 要检测的文件
:return:
"""
cmd = [
'cd', self.project_path, '&&', self.git_path, 'log', '-n', '1',
git_file_path, '|', 'sed', '-n', '\'2,1p\''
]
self.logger.debug('[GitOperator] {0}'.format(' '.join(cmd)))
result, _ = exec_cmd(' '.join(cmd))
if result:
result = result.strip().decode("utf-8").replace('Author:', '')
return result
def exec_match(self, cmd, match_str=None):
"""
:param cmd:
:param match_str:
:return:
"""
result = False
os.chdir("/tmp")
if isinstance(cmd, list):
cmd_str = '{0} {1}'.format(self._path, ' '.join(cmd))
else:
cmd_str = '{0} {1}'.format(self._path, cmd)
output, _ = exec_cmd(cmd_str)
if match_str and match_str in output.decode('utf-8'):
result = True
return result
def start_monitor(self):
"""
开始发送心跳监控
:return:
"""
if conf.agent.monitor_url:
logger.info(
'Send registration information to the SeeCode server, url: "{0}".'
.format(conf.agent.monitor_url))
# 节点注册
registration_data = {
'hostname': get_hostname(),
'ipv4': '127.0.0.1',
'role': 'client',
'client_version': self.version,
'action': 'node',
}
if conf.server.domain:
domain = urlparse(conf.server.domain).netloc
registration_data['ipv4'] = get_localhost_ip(domain=domain)
else:
registration_data['ipv4'] = get_localhost_ip()
self.request.post_data(conf.agent.monitor_url, registration_data)
# 服务心跳监控
if conf.general.services:
heartbeat_data = {
'hostname': get_hostname(),
'ipv4': registration_data['ipv4'],
'action': 'heartbeat',
'items': []
}
for item in conf.general.services:
try:
if item['keyword'] and item['role'] == 'client':
output, err = exec_cmd(
'ps -ef | grep "{0}" |grep -v grep|wc -l'.
format(item['keyword']))
if err:
logger.error(err)
else:
result = parse_int(output.strip())
if result == 0:
status = 2
else:
status = 1
heartbeat_data['items'].append({
'key':
item['key'],
'status':
status,
})
except Exception as ex:
logger.error(ex)
if heartbeat_data['items']:
services_url = '{0}{1}node/services/'.format(
conf.server.domain, conf.server.api_uri)
self.request.post_data(services_url, heartbeat_data)
else:
logger.critical(
'"monitor_url" parameter is empty, agent monitoring is not possible.'
)
def checkout_branch_sync_code(self, branch_name, sync_code=False):
"""
切换分支同步代码
:param branch_name:
:param sync_code:
:return:
"""
# 初始化并克隆代码
git_config_path = os.path.join(self.project_path, '.git')
if not os.path.isdir(git_config_path):
self.logger.info(
'[GitOperator] git clone project to "{0}" ...'.format(
self.project_path))
cmd = [
self.git_path, 'clone', '--depth 1', self.git_url,
self.project_path
]
self.logger.info(
"[GitOperator] Start executing commands: '{0}'".format(
' '.join(cmd)))
_, err = exec_cmd(' '.join(cmd))
if err and 'fatal: Could not read from remote repository.' in err.decode(
'utf-8'):
raise GitDoesNotPermissionException(
'克隆项目代码失败，当前没有权限。{0}'.format(err))
cmd = [
'cd',
self.project_path,
'&&',
self.git_path,
'remote',
'add',
'upstream',
self.git_url,
'&&',
self.git_path,
'fetch',
'--all',
'-p',
]
self.logger.info(
"[GitOperator] Start executing commands: '{0}'".format(
' '.join(cmd)))
exec_cmd(' '.join(cmd))
# 解决多个分支切换导致的文件冲突问题
cmd = [
'cd', self.project_path, '&&', self.git_path, 'reset', '.', '&&',
self.git_path, 'checkout', '.', '&&', self.git_path, 'stash'
]
self.logger.debug(
"[GitOperator] Start executing commands: '{0}'".format(
' '.join(cmd)))
exec_cmd(' '.join(cmd))
self.get_all_local_branch()
if branch_name in self.local_branch_list:
cmd = [
'cd', self.project_path, '&&', self.git_path, 'checkout',
branch_name
]
self.logger.debug(
"[GitOperator] Start executing commands: '{0}'".format(
' '.join(cmd)))
exec_cmd(' '.join(cmd))
if sync_code: # fetch && pull
self.logger.info("[GitOperator] Force update of local code...")
cmd = [
'cd', self.project_path, '&&', self.git_path, 'fetch', '--all',
'-p', '&&', self.git_path, 'pull'
]
self.logger.debug(
"[GitOperator] Start executing commands: '{0}'".format(
' '.join(cmd)))
exec_cmd(' '.join(cmd))
# 验证扫描分支是否一致
current_branch = self.get_current_branch()
if current_branch.strip() != branch_name.strip(): # 判断本地是否存在分支
self.logger.info(
'[GitOperator] Start switching branches, current branch: [{0}], target branch: [{1}].'
.format(current_branch.strip(), branch_name.strip()))
cmd = [
'cd', self.project_path, '&&', self.git_path, 'checkout', '-b',
branch_name, 'remotes/upstream/{0}'.format(branch_name)
]
self.logger.debug(
"[GitOperator] Start executing commands: '{0}'".format(
' '.join(cmd)))
_last_output, _ = exec_cmd(' '.join(cmd))
current_branch = self.get_current_branch()
if current_branch != branch_name.strip():
raise CheckoutBranchException('切换分支失败! {0}'.format(_))
self.logger.info('[GitOperator] Code synchronization completed.')
return True