def create_artifact_index_file(self, project_id=None, pipeline_id=None):
if project_id is None:
project_id = os.getenv('CI_PROJECT_ID')
if pipeline_id is None:
pipeline_id = os.getenv('CI_PIPELINE_ID')
gitlab_inst = gitlab_api.Gitlab(project_id)
artifact_index_list = []
for build_job_name in self.case_group.BUILD_JOB_NAMES:
job_info_list = gitlab_inst.find_job_id(build_job_name,
pipeline_id=pipeline_id)
for job_info in job_info_list:
parallel_num = job_info[
'parallel_num'] or 1 # Could be None if "parallel_num" not defined for the job
raw_data = gitlab_inst.download_artifact(
job_info['id'],
[self.format_build_log_path(parallel_num)])[0]
build_info_list = [
json.loads(line)
for line in raw_data.decode().splitlines()
]
for build_info in build_info_list:
build_info['ci_job_id'] = job_info['id']
artifact_index_list.append(build_info)
artifact_index_file = self.case_group.get_artifact_index_file()
try:
os.makedirs(os.path.dirname(artifact_index_file))
except OSError as e:
if e.errno != errno.EEXIST:
raise e
with open(artifact_index_file, 'w') as f:
json.dump(artifact_index_list, f)
def update_submodule(git_module_file, submodules_to_update):
gitlab_inst = gitlab_api.Gitlab()
submodules = []
with open(git_module_file, "r") as f:
data = f.read()
match = SUBMODULE_PATTERN.search(data)
while True:
next_match = SUBMODULE_PATTERN.search(data, pos=match.end())
if next_match:
end_pos = next_match.start()
else:
end_pos = len(data)
path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos)
url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos)
path = path_match.group(1)
url = url_match.group(1)
filter_result = True
if submodules_to_update:
if path not in submodules_to_update:
filter_result = False
if filter_result:
submodules.append(SubModule(gitlab_inst, path, url))
match = next_match
if not match:
break
shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True)
for submodule in submodules:
submodule.download_archive()
def create_artifact_index_file(project_id=None, pipeline_id=None):
if project_id is None:
project_id = os.getenv("CI_PROJECT_ID")
if pipeline_id is None:
pipeline_id = os.getenv("CI_PIPELINE_ID")
gitlab_inst = gitlab_api.Gitlab(project_id)
artifact_index_list = []
def format_build_log_path():
return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
for job_info in job_info_list:
raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
build_info_list = [json.loads(line) for line in raw_data.splitlines()]
for build_info in build_info_list:
build_info["ci_job_id"] = job_info["id"]
artifact_index_list.append(build_info)
try:
os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
except OSError:
# already created
pass
with open(ARTIFACT_INDEX_FILE, "w") as f:
json.dump(artifact_index_list, f)
def __init__(self, dest_root_path, artifact_index_file, app_path, config_name, target):
assert gitlab_api
# at least one of app_path or config_name is not None. otherwise we can't match artifact
assert app_path or config_name
assert os.path.exists(artifact_index_file)
self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID"))
self.dest_root_path = dest_root_path
with open(artifact_index_file, "r") as f:
artifact_index = json.load(f)
self.artifact_info = self._find_artifact(artifact_index, app_path, config_name, target)
def try_to_download_artifacts(bin_path: str) -> None:
'''
bin_path: "SSC/ssc_bin/ESP32[C3]/SSC[_APP]"
'''
project_id = os.getenv('CI_PROJECT_ID')
pipeline_id = os.getenv('CI_PIPELINE_ID')
gitlab_inst = gitlab_api.Gitlab(project_id)
build_job_name = SSC_BUILD_JOB_MAP[bin_path.split('/')[-2]]
job_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
files_to_download = [os.path.join(bin_path, f) for f in NEEDED_FILES]
for job_info in job_list:
try:
gitlab_inst.download_artifact(job_info['id'], files_to_download, IDF_PATH)
print('Downloaded {} from {}'.format(bin_path, job_info['id']))
break
except gitlab.exceptions.GitlabError as e:
if e.response_code == 404:
continue
raise
def download_all_test_results(result_path: str,
include_retried: bool = False) -> None:
if os.path.exists(result_path):
shutil.rmtree(result_path)
os.makedirs(result_path, exist_ok=True)
gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
_include_retried = 'true' if include_retried else 'false'
jobs = pipelines.jobs.list(all=True,
per_page=100,
include_retried=_include_retried)
job_info_list = []
for job in jobs:
if job.stage in ['target_test', 'host_test']:
log_path = ''
if job.status not in ['success', 'failed']:
print('Job {}({}) is not finished'.format(job.id, job.name))
elif not hasattr(job, 'artifacts_file'):
print('Job {}({}) has no artifacts.'.format(job.id, job.name))
else:
log_path = os.path.join(result_path, 'job_{}'.format(job.id))
print('Downloading artifacts from: {}'.format(job.name))
os.makedirs(log_path, exist_ok=True)
gitlab_inst.download_artifacts(job.id, log_path)
job_info = {
'id': job.id,
'name': job.name,
'tag_list': job.tag_list,
'status': job.status,
'stage': job.stage,
'web_url': job.web_url,
'commit_url': job.commit['web_url'],
'log_path': log_path,
}
job_info_list.append(job_info)
with open(os.path.join(result_path, 'index.json'), 'w') as f:
f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
def create_artifact_index_file(project_id=None,
pipeline_id=None,
case_group=ExampleGroup):
if project_id is None:
project_id = os.getenv("CI_PROJECT_ID")
if pipeline_id is None:
pipeline_id = os.getenv("CI_PIPELINE_ID")
gitlab_inst = gitlab_api.Gitlab(project_id)
artifact_index_list = []
def format_build_log_path():
parallel = job_info[
"parallel_num"] # Could be None if "parallel_num" not defined for the job
return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR,
parallel or 1)
for build_job_name in case_group.BUILD_JOB_NAMES:
job_info_list = gitlab_inst.find_job_id(build_job_name,
pipeline_id=pipeline_id)
for job_info in job_info_list:
raw_data = gitlab_inst.download_artifact(
job_info["id"], [format_build_log_path()])[0]
build_info_list = [
json.loads(line) for line in raw_data.splitlines()
]
for build_info in build_info_list:
build_info["ci_job_id"] = job_info["id"]
artifact_index_list.append(build_info)
artifact_index_file = get_artifact_index_file(case_group=case_group)
try:
os.makedirs(os.path.dirname(artifact_index_file))
except OSError:
# already created
pass
with open(artifact_index_file, "w") as f:
json.dump(artifact_index_list, f)
