def _update_obs_repo_rpms(self):
"""
update obs repo rpms
"""
log.debug("update repo rpms")
update_repo = RPMManager(**self.kwargs)
update_repo.update_pkgs()
def run(self):
"""
run main
return:
"""
if self.kwargs["check_codes"]:
self._check_codes()
elif self.kwargs["repo_rpms_update"]:
self._update_obs_repo_rpms()
elif self.kwargs["latest_info"]:
self._save_latest_info()
elif self.kwargs["repository"] == "obs_meta":
self._obs_meta_action()
elif self.kwargs["sync_gitee_to_obs"] == "true":
if not self.update_enabled_flag[self.kwargs["branch"]]:
if self.kwargs["repository"] and self.kwargs[
"repository"] not in self.ignore_list:
log.debug(
"can not update branch:%s, package: %s" %
(self.kwargs["branch"], self.kwargs["repository"]))
self._save_unsync_info()
else:
self._update_package()
elif self.kwargs["check_pkg_service"] == "true":
print("check_pkg_service")
check = CheckMetaPull(**self.kwargs)
check.do_all()
elif self.kwargs["get_latest_date"] == "true" and self.kwargs["branch"]:
self._get_latest_date()
elif self.kwargs["obs_mail_notice"] == "true":
self._mail_notice()
elif self.kwargs["pckg_mgmt"] == "true":
self._pckg_mgmt()
elif self.kwargs["check_pckg_mgmt"] == "true":
self._check_release_mgmt()
def _get_latest_date(self):
"""
get the latest git date to obs_pkgs_rpms
"""
log.debug("get latest date")
get_date = GETDate(**self.kwargs)
get_date.update_to_obs_pkg_rpms()
def _mail_notice(self):
"""
send mail to the package owner
"""
log.debug("OBS Mail Notice")
mail_notice = ObsMailNotice(**self.kwargs)
mail_notice.notify_all_respon_person()
def _get_repo_change_file(self, owner=None, pkgname=None, repo_path=None):
"""
Obtain the change for latest commit
"""
changed_file_cmd = "git diff --name-status HEAD~1 HEAD~0"
fetch_cmd = "git fetch origin pull/%s/head:thispr" % self.prid
checkout_cmd = "git checkout thispr"
get_fetch = None
if not repo_path:
self._clean(pkgname)
release_path = self._get_latest_git_repo(owner, pkgname)
self.manage_path = release_path
os.chdir(release_path)
else:
os.chdir(repo_path)
for x in range(5):
fetch_result = os.system(fetch_cmd)
checkout_result = os.system(checkout_cmd)
log.debug("STATUS:{0} and {1}".format(fetch_result, checkout_result))
if fetch_result == 0 and checkout_result == 0:
get_fetch = True
break
else:
os.chdir(self.current_path)
self._clean(pkgname)
path_release = self._get_latest_git_repo(owner, pkgname)
os.chdir(path_release)
self.manage_path = os.path.join(self.current_path, pkgname)
changed_file = os.popen(changed_file_cmd).readlines()
if get_fetch and changed_file:
log.info(changed_file)
return changed_file
else:
raise SystemExit("Error:can not obtain the content for this commit")
def _check_codes(self):
"""
check codes between gitee and obs
"""
log.debug("check codes")
check_codes = CheckCode(**self.kwargs)
check_codes.check_all()
def _pckg_mgmt(self):
"""
sync obs_meta according to the pckg-mgmt.yaml
"""
log.debug("Sync Pckg Mgmt")
mgmt = SyncPckgMgmt(**self.kwargs)
mgmt.sync_yaml_meta()
def _save_latest_info(self):
"""
save package latest info
return:
"""
log.debug("save latest info")
si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"])
si.save_latest_info(self.kwargs["branch"])
def _update_package(self):
"""
update package code for obs
return:
"""
log.debug("update package code")
syc = SYNCCode(**self.kwargs)
syc.sync_code_to_obs()
def _save_unsync_info(self):
"""
save unsync package info for manual operation later
return:
"""
log.debug("save package info")
si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"])
si.save_unsync_package(self.kwargs["repository"],
self.kwargs["branch"])
def update_repos_db(self):
"""
update obs repos db
"""
cmd = "chown -R obsrun:obsrun %s/%s/%s/:full; obs_admin --rescan-repository %s %s %s" \
% (self.obs_project_root_path, self.rpms_to_repo_path, self.arch, \
self.rpms_to_repo_path.split("/")[0], self.rpms_to_repo_path.split("/")[1], self.arch)
log.debug(cmd)
ret = self.pex.ssh_cmd(cmd)
def copy_new_rpm(self, pkg):
"""
copy rpm to repo
pkg: name of package
rpm: name of rpm file of pkg
"""
cmd = "cp -p %s/%s/%s/%s/%s/*.rpm %s/%s/%s/:full/ && rm -rf %s/%s/%s/:full/*.src.rpm" \
% (self.obs_project_root_path, self.obs_project, self.repo, \
self.arch, pkg, self.obs_project_root_path, \
self.rpms_to_repo_path, self.arch, self.obs_project_root_path, self.rpms_to_repo_path, self.arch)
log.debug("%s: %s" % (pkg, cmd))
self.pex.ssh_cmd(cmd)
def _obs_meta_action(self):
"""
action basis on change of obs_meta
return:
"""
log.debug("obs_meta change")
if not self.kwargs["obs_meta_path"]:
log.error("can not find obs_meta path")
else:
obs_prjm = OBSPrjManager(self.kwargs["obs_meta_path"])
obs_prjm.manager_action()
obs_pkgm = OBSPkgManager(**self.kwargs)
obs_pkgm.obs_pkg_admc()
def _copy_package(self, pkg, from_path, to_path):
"""
copy package from from_path to to_path
from_path: path of package that will be copied
to_path: path fo package that will go
"""
if self.kwargs["branch"] == "master":
cmd = "cp -r %s/%s %s && sed -i 's/openEuler/%s/g' %s/%s/_service" % \
(from_path, pkg, to_path, self.kwargs["branch2"], to_path, pkg)
else:
cmd = "cp -r %s/%s %s && sed -i 's/%s/%s/g' %s/%s/_service" % \
(from_path, pkg, to_path, self.kwargs["branch"], \
self.kwargs["branch2"], to_path, pkg)
ret = os.popen(cmd).read()
log.debug(ret)
def _git_clone(self, git_house):
"""
git clone function
"""
if git_house == "obs_meta":
os.system(
"cd %s && git diff --name-status HEAD~1 HEAD~0 > %s && cd -" %
(self.obs_meta_path, self.patch_file_path))
if git_house == "community":
community_path = os.path.join(self.init_path, "community")
if os.path.exists(community_path):
shutil.rmtree(community_path)
git_url = "https://%s:%[email protected]/openeuler/community.git" % (
self.giteeUserName, self.giteeUserPwd)
ret = os.popen("git lfs --depth 1 clone %s %s" %
(git_url, community_path)).read()
log.debug(ret)
def _save_latest_info_by_pkg(self, datestr_root_path, prj, pkg, file_list,
f_csv):
"""
save latest info of package
"""
if pkg in file_list:
cmd = "cat %s/%s" % (datestr_root_path, pkg)
log.debug(cmd)
timestr = os.popen(cmd).read().replace("\n", "")
else:
timestr = 0
cmd = "osc list -b %s %s|grep rpm" % (prj, pkg)
log.debug(cmd)
rpms = ' '.join(
list(
set(os.popen(cmd).read().replace(" ", "").split("\n")) -
set([''])))
f_csv.writerow([timestr, pkg, rpms])
def _copy_packages(self):
"""
copy some packages from obs project A to project B
"""
if self.kwargs["branch2"] and self.kwargs["project2"]:
os.chdir(self.kwargs["obs_meta_path"])
from_path = os.path.join(self.kwargs["branch"],
self.kwargs["project"])
to_path = os.path.join(self.kwargs["branch2"],
self.kwargs["project2"])
pkgs = self.kwargs["pkglist"]
if not pkgs:
pkgs = os.listdir(from_path)
for pkg in pkgs:
self._copy_package(pkg, from_path, to_path)
cmd = "git add %s && git commit -m 'add pkgs to project %s' && git push" % \
(to_path, self.kwargs["project2"])
ret = os.popen(cmd).read()
log.debug(ret)
def sync_code_to_obs(self):
"""
The only way to offer that make all fuction runing
"""
if not self.pkgs:
if self.repository and self.gitee_branch:
if not self._pre_sync_code(self.project):
raise SystemExit("SYNC %s ERROR" % self.repository)
else:
raise SystemExit('please check you arguments')
else:
if "broken" == self.pkgs[0] and len(self.pkgs) == 1:
project_flag = "yes"
broken_cmd = "osc r --csv %s -r standard_aarch64 -a aarch64 2>/dev/null | \
grep broken | awk -F ';' '{print $1}'" % self.project
log.info("Get the broken rpm:%s" % broken_cmd)
broken_result = os.popen(broken_cmd).read()
log.debug(broken_result)
broken_list = broken_result.split('\n')
self.pkgs = [x for x in broken_list if x != '']
log.info("broken_rpmlist: %s" % self.pkgs)
if not self.pkgs:
log.info("There are no broken pkgs in %s" % self.project)
if "All" == self.pkgs[0] and len(self.pkgs) == 1:
project_flag = "yes"
if not self.repository and self.project:
cmd = "osc ls %s 2>/dev/null" % self.project
pkgs = os.popen(cmd).read().split('\n')
self.pkgs = [x for x in pkgs if x != '']
log.info("project_rpmlist: %s" % self.pkgs)
if not self.pkgs:
log.info("There are no pkgs in %s" % self.project)
for pkg in self.pkgs:
if "\n" in pkg:
self.repository = pkg.replace('\n', '')
else:
self.repository = pkg
sync_result = self._pre_sync_code(self.project)
if not sync_result:
self.sync_failed_rpms.append(self.repository)
if self.sync_failed_rpms:
log.error("SYNC ERROR LIST: %s" % ",".join(self.sync_failed_rpms))
raise SystemExit("Failed, There are some pkgs sync failure")
def _push_to_pkg_rpms(self):
"""
push the latest obs_pkg_rpms to origin
"""
os.chdir("obs_pkg_rpms")
status_cmd = "git status -s"
commit_cmd = "git add -A && git commit -m 'update for package'"
if os.popen(status_cmd).read():
if os.system(commit_cmd) == 0:
for i in range(5):
push_cmd = "git push -f"
push_result = subprocess.getstatusoutput(push_cmd)
log.info(push_result)
if push_result[0] == 0:
log.info("SUCCESS:Push success for latest date to obs_pkg_rpms")
return
else:
log.debug("Try Push to obs_pkg_rpms: %s" % i)
raise SystemExit("Failed: Push to obs_pkg_rpms failed")
else:
log.info("NO CHAGE,nothing to commit")
def _get_diff_msg(self, old_msg, new_msg, change_file_list):
"""
Get the content of this submission
"""
change_list = {}
for change in change_file_list:
change_list[change] = []
for new in new_msg[change]:
if new not in old_msg[change]:
change_list[change].append(new)
for change in change_file_list:
if change_list[change]:
log.debug("Change in {0}:".format(change))
for msg in change_list[change]:
log.debug(msg)
else:
del change_list[change]
log.info("The are no new msg in {0}!!!".format(change))
if change_list:
return change_list
else:
log.info("The are no new msg in your yaml!!!")
sys.exit()
def save_latest_info(self, branch_name):
"""
save latest info of package, include update time of package code and rpms
"""
self._download()
try:
datestr_root_path = os.path.join(self.obs_pkg_rpms_files_dir,
branch_name)
if not os.path.exists(datestr_root_path):
os.makedirs(datestr_root_path)
latest_rpm_file = os.path.join(self.obs_pkg_rpms_files_dir,
"latest_rpm",
"%s.csv" % branch_name)
with open(latest_rpm_file, "w") as f:
f_csv = csv.writer(f)
file_list = os.listdir(datestr_root_path)
prj_list = self.branch_prj[branch_name].split(" ")
log.debug(prj_list)
var_list = []
for prj in prj_list:
cmd = "osc list %s" % prj
log.debug(cmd)
pkgs = os.popen(cmd).read().split("\n")
for pkg in pkgs:
var_list.append(
([datestr_root_path, prj, pkg, file_list,
f_csv], None))
try:
pool = threadpool.ThreadPool(20)
requests = threadpool.makeRequests(
self._save_latest_info_by_pkg, var_list)
for req in requests:
pool.putRequest(req)
pool.wait()
except KeyboardInterrupt as e:
log.error(e)
cmd = "sort -u -r %s > tmp && cat tmp > %s" % (latest_rpm_file,
latest_rpm_file)
if os.system(cmd) != 0:
log.error("sort file fail")
cmd = "cd %s && git pull && git add * && git commit -m 'update file' && git push" % \
self.obs_pkg_rpms_files_dir
log.debug(cmd)
for i in range(5):
if os.system(cmd) == 0:
break
except AttributeError as e:
log.error(e)
finally:
self._delete()
def update_pkg(self, pkg):
"""
update one package
pkg: name of package
"""
if pkg in self.old_pkg_rpms.keys():
old_rpms_list = self.old_pkg_rpms[pkg]
old_rpms_list.sort()
else:
old_rpms_list = None
new_rpms_list = self.get_new_rpms_by_pkg(pkg)
new_rpms_list.sort()
if self.kwargs["all"] == "True" and new_rpms_list:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
elif old_rpms_list != new_rpms_list and new_rpms_list:
try:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
except Exception as e:
self.backup_old_rpms_by_pkg(pkg, new_rpms_list)
elif self.kwargs["compare"] == "True":
if not self.compare_rpms(pkg):
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
else:
log.debug("%s all rpms are latest should do nothing" % pkg)
if new_rpms_list:
for i in range(5):
try:
if not self.rpms_exists(new_rpms_list):
log.debug(
"check %s rpms not exists, shoud copy again" % pkg)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
else:
log.debug("check %s all rpms exists" % pkg)
break
except Exception as e:
log.error(e)
else:
cmd = "osc list %s | grep ^%s" % (self.obs_project, pkg)
ret = os.popen(cmd).read().split("\n")
if pkg not in ret:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.old_pkg_rpms[pkg] = new_rpms_list
