def __init__(self, **kwargs):
"""
obs_project: obs project name
repo: obs project repo where store all packages
arch:
pkgs:
"""
self.kwargs = kwargs
par = ParserConfigIni()
self.obs_project_repo_dict = par.get_obs_repos_dict()
self.obs_project_root_path = par.get_obs_prj_root_path()
self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"]
self.obs_project = self.kwargs["project"]
self.rpms_to_repo_path = None
self.old_pkg_rpms = None
self.repo = self.kwargs["repo"]
self.arch = self.kwargs["arch"]
self.pkgs = self.kwargs["pkglist"]
self.compare = self.kwargs["compare"]
self.old_pkg_rpms = {}
self._set_rpms_to_repo()
self.pex = Pexpect(self.kwargs["repo_server_user"],
self.kwargs["repo_server_ip"],
self.kwargs["repo_server_pwd"],
self.kwargs["repo_server_port"])
log.info(self.rpms_to_repo_path)
self.obs_pkg_rpms_files_dir = None
self._download_obs_pkg_rpms_file(self.obs_pkg_rpms_url, self.kwargs["gitee_user"], \
self.kwargs["gitee_pwd"])
self.obs_pkg_rpms_file = os.path.join(self.obs_pkg_rpms_files_dir, "repo_files", \
"%s_%s.yaml" % (self.obs_project.replace(":", "-"), self.arch))
self.get_old_rpms_list_from_file(self.obs_pkg_rpms_file)
def __init__(self, **kwargs):
"""
kargs: dict, init dict by 'a': 'A' style
The dict key_value as the following:
repository: The package you want sync it to obs
gitee_branch: The branch for you commit in gitee
giteeuser: The gitee account
giteeuserpwd: The gitee passwd
meta_path: Where the obs_meta save in jenkins
obs_server_user: The user name for you use in server
obs_server_ip: The ip for you server
obs_server_passwd: The password for your ip
obs_server_port: The port for your ip
"""
self.timestr = time.strftime("%Y%m%d %H-%M-%S", time.localtime())
self.kwargs = kwargs
self.repository = self.kwargs['repository']
self.gitee_branch = self.kwargs['branch']
self.giteeuser = self.kwargs['gitee_user']
self.giteeuserpwd = self.kwargs['gitee_pwd']
self.meta_path = self.kwargs['obs_meta_path']
self.pkgs = self.kwargs['pkglist']
self.project = self.kwargs['project']
self.cmd = Pexpect(self.kwargs['source_server_user'],
self.kwargs['source_server_ip'],
self.kwargs['source_server_pwd'],
self.kwargs['source_server_port'])
par = ParserConfigIni()
self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"]
self.obs_pkg_prms_files_dir = None
self.sync_failed_rpms = []
def __init__(self, **kwargs):
"""
kwargs: dict, init dict by 'a': 'A' style
The dict key_value as the following:
branch: which you wangt to get the package date from
source_server_user: The user name for you use in server
source_server_pwd: The password for your ip
source_server_port: The port for your ip
giteeuser: The gitee account
giteeuserpwd: The gitee passwd
"""
self.branch = kwargs["branch"]
self.cmd = Pexpect(kwargs["source_server_user"],
kwargs["source_server_ip"],
kwargs["source_server_pwd"],
kwargs["source_server_port"])
self.giteeuser = kwargs["gitee_user"]
self.giteeuserpwd = kwargs["gitee_pwd"]
self.sourcedir = "/srv/cache/obs/tar_scm/repo/next/"
self.pkg_time = {}
self.realdir = None
class SYNCCode(object):
"""
if the rpm package has changed in gitee what you should donext
(Synchronize the latest code to OBS)
"""
def __init__(self, **kwargs):
"""
kargs: dict, init dict by 'a': 'A' style
The dict key_value as the following:
repository: The package you want sync it to obs
gitee_branch: The branch for you commit in gitee
giteeuser: The gitee account
giteeuserpwd: The gitee passwd
meta_path: Where the obs_meta save in jenkins
obs_server_user: The user name for you use in server
obs_server_ip: The ip for you server
obs_server_passwd: The password for your ip
obs_server_port: The port for your ip
"""
self.timestr = time.strftime("%Y%m%d %H-%M-%S", time.localtime())
self.kwargs = kwargs
self.repository = self.kwargs['repository']
self.gitee_branch = self.kwargs['branch']
self.giteeuser = self.kwargs['gitee_user']
self.giteeuserpwd = self.kwargs['gitee_pwd']
self.meta_path = self.kwargs['obs_meta_path']
self.pkgs = self.kwargs['pkglist']
self.project = self.kwargs['project']
self.cmd = Pexpect(self.kwargs['source_server_user'],
self.kwargs['source_server_ip'],
self.kwargs['source_server_pwd'],
self.kwargs['source_server_port'])
par = ParserConfigIni()
self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"]
self.obs_pkg_prms_files_dir = None
self.sync_failed_rpms = []
def _write_date_to_file(self):
"""
write date repository changed to file
"""
tmpdir = os.popen("mktemp").read().split("\n")[0]
self.obs_pkg_prms_files_dir = git_repo_src(self.obs_pkg_rpms_url, self.giteeuser, self.giteeuserpwd, tmpdir)
try:
branch_path = os.path.join(self.obs_pkg_prms_files_dir, self.gitee_branch)
if not os.path.exists(branch_path):
os.makedirs(branch_path)
cmd1 = "echo %s > %s/%s" % (self.timestr, branch_path, self.repository)
if os.system(cmd1) != 0:
log.error("fail to write date of package changed to file")
cmd2 = "cd %s && git pull && git add * && git commit -m 'update date for pkg %s' && git push"\
% (self.obs_pkg_prms_files_dir, self.repository)
if os.system(cmd2) != 0:
log.error("fail to update file to %s" % self.repository)
except AttributeError as e:
log.error(e)
finally:
cmd = "rm -rf %s" % tmpdir
os.system(cmd)
def _git_clone(self, rpm_dir, gitee_branch, path):
"""
rpm_dir: The name for git repository "it will exist under the current directory"
rpm_branch: The branch you want to checkout for this git repository
"""
log.info("Git clone the %s in %s" % (rpm_dir, gitee_branch))
self.cmd.ssh_cmd("rm -rf %s " % (path))
repo_url = "https://%s:%[email protected]/src-openEuler/%s" % (self.giteeuser, self.giteeuserpwd, rpm_dir)
for i in range(5):
clone_result = self.cmd.ssh_cmd("git lfs clone --depth=1 %s -b %s %s"
% (repo_url, gitee_branch, path), 600)
pull_result_last = str(self.cmd.ssh_cmd('git -C %s pull' % path)[1].strip()).split("'")[1]
if "Already" in pull_result_last:
log.info(pull_result_last)
log.info("At now %s the branch is in %s" % (rpm_dir, gitee_branch))
return True
else:
log.info("_GIT_CLONE: %s" % i)
clear_repo = self.cmd.ssh_cmd("rm -rf %s" % path)
log.info("clear_repo:%s" % clear_repo)
continue
log.error("_GIT_CLONE_ERROR: %s" % rpm_dir)
return False
def _get_obs_project(self):
"""
Get the obs project from gitee_branch
"""
log.info("Start get the obs_project")
if "Multi-Version" in self.gitee_branch:
path = self.meta_path + '/' + 'multi_version/' + self.gitee_branch
log.info(path)
cmd = "find %s -name %s | awk -F '/' '{print $5}'" % (path, self.repository)
else:
path = self.meta_path + '/' + self.gitee_branch
log.info(path)
cmd = "find %s -name %s | awk -F '/' '{print $4}'" % (path, self.repository)
all_project = os.popen(cmd).readlines()
log.info(all_project)
obs_project = None
for project in all_project:
obs_project = project.replace('\n', '')
if ":Bak" in obs_project:
continue
else:
break
if obs_project and ":Bak" not in obs_project:
log.info("The %s obs_project for gitee_%s is <<<<%s>>>>" % (
self.repository, self.gitee_branch, obs_project))
return obs_project
else:
log.error("Failed !!! The rpm is not exist in %s branch !!!"
% self.gitee_branch)
return False
def _get_latest_gitee_pull(self):
"""
make the obs server code be the latest
"""
if self.gitee_branch == "master":
source_path = "/srv/cache/obs/tar_scm/repo/next/openEuler"
else:
source_path = "/srv/cache/obs/tar_scm/repo/next/" + self.gitee_branch
if self.repository == "CreateImage":
log.error("The <<CreateImage>> packages does not need to be sync!!!")
return False
elif self.repository == "kernel":
tag_path = "%s/kernel/SOURCE" % source_path
ssh_cmd = "if [ -d %s ];then echo 'exist';else echo 'need to clone';fi" % tag_path
repository_exist = self.cmd.ssh_cmd(ssh_cmd)
repository_exist = str(repository_exist[1].strip()).split("'")[1]
if repository_exist != "exist":
clone_result = self._git_clone(self.repository, self.gitee_branch, f"{source_path}/kernel")
pull_result = str(self.cmd.ssh_cmd("git -C %s/kernel pull" % source_path)
[1].strip()).split("'")[1]
log.info(pull_result)
kernel_tags = str(self.cmd.ssh_cmd("cat %s" % tag_path)[1].strip()).split("'")[1]
log.info(kernel_tags)
open_kernel_path = "%s/openEuler-kernel/kernel" % source_path
open_kernel_git = "https://%s:%[email protected]/openeuler/kernel" % (self.giteeuser, self.giteeuserpwd)
log.info(open_kernel_git)
ssh_rm_result = self.cmd.ssh_cmd("rm -rf %s" % open_kernel_path)
log.info(ssh_rm_result)
for i in range(5):
ssh_clone_result = self.cmd.ssh_cmd("git lfs clone --depth=1 %s -b %s %s" % (open_kernel_git,
kernel_tags, open_kernel_path), 600)
pull_result_last = str(self.cmd.ssh_cmd('git -C %s pull' % open_kernel_path)[1].strip()).split("'")[1]
if "Already" in pull_result_last:
log.info(pull_result_last)
log.info("kernel gitee pull success")
return True
else:
clear_repo = self.cmd.ssh_cmd("rm -rf %s" % open_kernel_path)
log.info("clear_repo: %s" % clear_repo)
continue
log.error("kernel gitee pull failed")
return False
else:
rpm_path = source_path + '/' + self.repository
ssh_cmd = "if [ -d %s ];then echo 'exist';else echo 'need to clone';fi" % rpm_path
repository_exist = self.cmd.ssh_cmd(ssh_cmd)
repository_exist = str(repository_exist[1].strip()).split("'")[1]
log.info(self.repository + ':' + repository_exist)
if repository_exist == 'exist':
pull_result = str(self.cmd.ssh_cmd('git -C %s pull' % rpm_path)[1].strip()).split("'")[1]
log.info(pull_result)
pull_result_last = str(self.cmd.ssh_cmd('git -C %s pull' % rpm_path)[1].strip()).split("'")[1]
if "Already" in pull_result_last:
log.info(pull_result_last)
return True
else:
clone_result = self._git_clone(self.repository, self.gitee_branch, rpm_path)
return clone_result
else:
clone_result = self._git_clone(self.repository, self.gitee_branch, rpm_path)
return clone_result
def _gitee_pr_to_obs(self, obs_pro):
"""
obs_pro:The obs project that gitee branch corresponds
"""
osc_service = os.popen("osc service remoterun %s %s" % (obs_pro, self.repository)).readlines()
for results in osc_service:
if "ok" in results:
log.info(results)
log.info("Success for osc service remoterun the %s" % self.repository)
return True
else:
continue
log.error("Failed !!! fail to service remoterun !!!")
return False
def _pre_sync_code(self, project=None):
"""
The way to offer that make all fuction runing
"""
if not project:
obs_project = self._get_obs_project()
else:
obs_project = project
log.info("The %s in %s" % (self.repository, obs_project))
pull_result = self._get_latest_gitee_pull()
remoterun_result = self._gitee_pr_to_obs(obs_project)
if obs_project and pull_result and remoterun_result:
log.info("SYNC %s in %s SUCCESS" % (self.repository, obs_project))
return True
else:
log.error("SYNC %s in %s ERROR: Please check the log.error" % (self.repository, obs_project))
return False
def sync_code_to_obs(self):
"""
The only way to offer that make all fuction runing
"""
if not self.pkgs:
if self.repository and self.gitee_branch:
if not self._pre_sync_code(self.project):
raise SystemExit("SYNC %s ERROR" % self.repository)
else:
raise SystemExit('please check you arguments')
else:
if "broken" == self.pkgs[0] and len(self.pkgs) == 1:
project_flag = "yes"
broken_cmd = "osc r --csv %s -r standard_aarch64 -a aarch64 2>/dev/null | \
grep broken | awk -F ';' '{print $1}'" % self.project
log.info("Get the broken rpm:%s" % broken_cmd)
broken_result = os.popen(broken_cmd).read()
log.debug(broken_result)
broken_list = broken_result.split('\n')
self.pkgs = [x for x in broken_list if x != '']
log.info("broken_rpmlist: %s" % self.pkgs)
if not self.pkgs:
log.info("There are no broken pkgs in %s" % self.project)
if "All" == self.pkgs[0] and len(self.pkgs) == 1:
project_flag = "yes"
if not self.repository and self.project:
cmd = "osc ls %s 2>/dev/null" % self.project
pkgs = os.popen(cmd).read().split('\n')
self.pkgs = [x for x in pkgs if x != '']
log.info("project_rpmlist: %s" % self.pkgs)
if not self.pkgs:
log.info("There are no pkgs in %s" % self.project)
for pkg in self.pkgs:
if "\n" in pkg:
self.repository = pkg.replace('\n', '')
else:
self.repository = pkg
sync_result = self._pre_sync_code(self.project)
if not sync_result:
self.sync_failed_rpms.append(self.repository)
if self.sync_failed_rpms:
log.error("SYNC ERROR LIST: %s" % ",".join(self.sync_failed_rpms))
raise SystemExit("Failed, There are some pkgs sync failure")
"""
import os
import sys
import pytest
current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0])
sys.path.append(os.path.join(current_path, ".."))
from func import SetEnv, CommonFunc
from core.gitee_to_obs import SYNCCode
from common.common import Pexpect
S = SetEnv()
C = CommonFunc()
repo_path = None
P = None
source_path = "/srv/cache/obs/tar_scm/repo/next/"
scmd = Pexpect("root", "127.0.0.1", "112233", "2222")
class TestCase(object):
def setup_class(self):
S.set_oscrc()
S.set_gitee()
global repo_path
repo_path = C.pull_from_gitee_repo(S.gitee_info["user"], S.gitee_info["passwd"], \
"https://gitee.com/{0}/obs_meta".format(S.gitee_info["user"]), "master", "obs_meta")
def teardown_class(self):
S.unset_oscrc()
S.unset_gitee()
cmd = "rm -rf %s" % repo_path
if os.system(cmd) == 0:
class RPMManager(object):
"""
update for obs repo rpms
"""
def __init__(self, **kwargs):
"""
obs_project: obs project name
repo: obs project repo where store all packages
arch:
pkgs:
"""
self.kwargs = kwargs
par = ParserConfigIni()
self.obs_project_repo_dict = par.get_obs_repos_dict()
self.obs_project_root_path = par.get_obs_prj_root_path()
self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"]
self.obs_project = self.kwargs["project"]
self.rpms_to_repo_path = None
self.old_pkg_rpms = None
self.repo = self.kwargs["repo"]
self.arch = self.kwargs["arch"]
self.pkgs = self.kwargs["pkglist"]
self.compare = self.kwargs["compare"]
self.old_pkg_rpms = {}
self._set_rpms_to_repo()
self.pex = Pexpect(self.kwargs["repo_server_user"],
self.kwargs["repo_server_ip"],
self.kwargs["repo_server_pwd"],
self.kwargs["repo_server_port"])
log.info(self.rpms_to_repo_path)
self.obs_pkg_rpms_files_dir = None
self._download_obs_pkg_rpms_file(self.obs_pkg_rpms_url, self.kwargs["gitee_user"], \
self.kwargs["gitee_pwd"])
self.obs_pkg_rpms_file = os.path.join(self.obs_pkg_rpms_files_dir, "repo_files", \
"%s_%s.yaml" % (self.obs_project.replace(":", "-"), self.arch))
self.get_old_rpms_list_from_file(self.obs_pkg_rpms_file)
def _set_rpms_to_repo(self):
"""
set rpms repo where new rpm go
"""
rpms_to_repo_list = self.obs_project_repo_dict[self.obs_project].split(
" ")
for rpms_to_repo in rpms_to_repo_list:
if rpms_to_repo.endswith(self.arch):
self.rpms_to_repo_path = rpms_to_repo
break
def _download_obs_pkg_rpms_file(self, url, gitee_user, gitee_pwd):
"""
download file by gitee repo
"""
tmpdir = os.popen("mktemp").read().split("\n")[0]
self.obs_pkg_rpms_files_dir = git_repo_src(url,
gitee_user,
gitee_pwd,
dest_dir=tmpdir)
def get_old_rpms_list_from_file(self, file_path):
"""
get old pkg rpms dict
file_path: yaml file which store all packages's rpms
"""
if os.path.exists(file_path):
with open(file_path, "r") as f:
self.old_pkg_rpms = yaml.load(f, Loader=yaml.FullLoader)
else:
log.error("no file for get rpms")
def get_new_rpms_by_pkg(self, pkg):
"""
get new rpms by package name
pkg: name of packages
"""
rpm_list = []
try:
cmd = "ls %s/%s/%s/%s/%s | grep 'rpm' | grep -v 'src.rpm'" \
% (self.obs_project_root_path, self.obs_project, self.repo, self.arch, pkg)
ret = self.pex.ssh_cmd(cmd)
for p in ret:
p = str(p, encoding='utf8')
if "rpm" in p:
rpm_list.append(p.replace("\r\n", ""))
except ValueError as e:
log.error(e)
except SystemError as e:
log.error(e)
except TypeError as e:
log.error(e)
return rpm_list
#def backup_old_rpm(self, pkg_bak, rpm):
def backup_old_rpm(self, backup_dir, pkg, pkg_bak, rpms_list):
"""
backup rpm
pkg_bak: dir for backup
rpm: name of rpm
"""
tmp_list = [
rpms_list[i:i + 500] for i in range(0, len(rpms_list), 500)
]
for l in tmp_list:
cmd = """
rm -r %s/%s-*; mkdir -p %s
rpms="%s"
for r in $rpms
do
mv %s/%s/%s/:full/$r %s/
done
""" % (backup_dir, pkg, pkg_bak, ' '.join(l), self.obs_project_root_path, \
self.rpms_to_repo_path, self.arch, pkg_bak)
self.pex.ssh_cmd(cmd)
def backup_old_rpms_by_pkg(self, pkg, rpms_list):
"""
backup old rpms by package name
pkg: name of packages
rpms_list: all rpms of package, type is list
"""
try:
t = time.strftime("%Y-%m-%d-%H-%M", time.localtime())
backup_dir = os.path.join(self.obs_project_root_path,
self.rpms_to_repo_path, "backup")
pkg_bak = os.path.join(backup_dir, "%s-%s" % (pkg, t))
self.backup_old_rpm(backup_dir, pkg, pkg_bak, rpms_list)
except ValueError as e:
log.error(e)
return False
except SystemError as e:
log.error(e)
return False
except TypeError as e:
log.error(e)
return False
except KeyError as e:
log.error(e)
return False
return True
def copy_new_rpm(self, pkg):
"""
copy rpm to repo
pkg: name of package
rpm: name of rpm file of pkg
"""
cmd = "cp -p %s/%s/%s/%s/%s/*.rpm %s/%s/%s/:full/ && rm -rf %s/%s/%s/:full/*.src.rpm" \
% (self.obs_project_root_path, self.obs_project, self.repo, \
self.arch, pkg, self.obs_project_root_path, \
self.rpms_to_repo_path, self.arch, self.obs_project_root_path, self.rpms_to_repo_path, self.arch)
log.debug("%s: %s" % (pkg, cmd))
self.pex.ssh_cmd(cmd)
def copy_new_rpms_to_repo(self, pkg, rpms_list):
"""
copy new rpms by package name to repo
pkg: name of package
rpms_list: all rpms of package, type is list
"""
self.copy_new_rpm(pkg)
self.old_pkg_rpms[pkg] = rpms_list
def rpms_exists(self, rpms_list):
"""
check rpms exists
rpms_list:
"""
try:
tmp_list = [
rpms_list[i:i + 500] for i in range(0, len(rpms_list), 500)
]
for l in tmp_list:
cmd = """
for r in %s
do
ls %s/%s/%s/:full/ | grep "$r"
if [ $? -ne 0 ];then
echo "notfind"
break
else
echo "find"
fi
done
""" % (' '.join(l), self.obs_project_root_path, self.rpms_to_repo_path,
self.arch)
ret = self.pex.ssh_cmd(cmd)
if "notfind" in str(ret) or "find" not in str(ret):
return False
except Exception as e:
log.error(e)
return False
return True
def compare_rpms(self, pkg):
"""
compare new rpm and old rpm by --dump
"""
old_rpm_path = f"{self.obs_project_root_path}/{self.rpms_to_repo_path}/{self.arch}/:full"
new_rpm_path = f"{self.obs_project_root_path}/{self.obs_project}/{self.repo}/{self.arch}/{pkg}"
cmd = """
rpms=`ls %s | grep '\.rpm' | grep -v '\.src.rpm' `
for r in $rpms
do
new_rpm_sha=`rpm -q --dump %s/$r | awk '{print \$4}'`
old_rpm_sha=`rpm -q --dump %s/$r | awk '{print \$4}'`
if [ "${new_rpm_sha}" != "${old_rpm_sha}" ];then
echo "dump notsame"
break
fi
new_rpm_requires=`rpm -pq --requires --nosignature %s/$r`
old_rpm_requires=`rpm -pq --requires --nosignature %s/$r`
if [ "${new_rpm_requires}" != "${old_rpm_requires}" ];then
echo "requires notsame"
break
fi
new_rpm_provides=`rpm -pq --provides --nosignature %s/$r`
old_rpm_provides=`rpm -pq --provides --nosignature %s/$r`
if [ "${new_rpm_provides}" != "${old_rpm_provides}" ];then
echo "provides notsame"
break
fi
done
""" % (new_rpm_path, new_rpm_path, old_rpm_path, new_rpm_path,
old_rpm_path, new_rpm_path, old_rpm_path)
ret = self.pex.ssh_cmd(cmd)
if "notsame" in str(ret):
if 'dump' in str(ret):
log.warning(f"Dump of {pkg} rpms is not same!")
elif 'requires' in str(ret):
log.warning(f"Requires of {pkg} rpms is not same!")
elif 'provides' in str(ret):
log.warning(f"Provides of {pkg} rpms is not same!")
return False
else:
log.info(f"Both of dump/requires/provides of {pkg} rpms are same!")
return True
def update_pkg(self, pkg):
"""
update one package
pkg: name of package
"""
if pkg in self.old_pkg_rpms.keys():
old_rpms_list = self.old_pkg_rpms[pkg]
old_rpms_list.sort()
else:
old_rpms_list = None
new_rpms_list = self.get_new_rpms_by_pkg(pkg)
new_rpms_list.sort()
if self.kwargs["all"] == "True" and new_rpms_list:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
elif old_rpms_list != new_rpms_list and new_rpms_list:
try:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
except Exception as e:
self.backup_old_rpms_by_pkg(pkg, new_rpms_list)
elif self.kwargs["compare"] == "True":
if not self.compare_rpms(pkg):
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
else:
log.debug("%s all rpms are latest should do nothing" % pkg)
if new_rpms_list:
for i in range(5):
try:
if not self.rpms_exists(new_rpms_list):
log.debug(
"check %s rpms not exists, shoud copy again" % pkg)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
else:
log.debug("check %s all rpms exists" % pkg)
break
except Exception as e:
log.error(e)
else:
cmd = "osc list %s | grep ^%s" % (self.obs_project, pkg)
ret = os.popen(cmd).read().split("\n")
if pkg not in ret:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.old_pkg_rpms[pkg] = new_rpms_list
def update_pkgs(self):
"""
update packages
"""
if not self.pkgs:
self.pkgs = list(set(list(set(os.popen("osc list %s" % self.obs_project).read().split("\n")) - set([''])) \
+ list(self.old_pkg_rpms.keys())))
pool = threadpool.ThreadPool(30)
requests = threadpool.makeRequests(self.update_pkg, self.pkgs)
for req in requests:
pool.putRequest(req)
pool.wait()
self.write_new_pkg_rpms_to_file()
self.update_repos_db()
def write_new_pkg_rpms_to_file(self):
"""
write new pkg rpms to yaml and git push for storing
"""
with open(self.obs_pkg_rpms_file, "w", encoding="utf-8") as f:
yaml.dump(self.old_pkg_rpms, f)
for i in range(5):
cmd = "cd %s && git add %s && git commit -m 'update rpms in file %s' \
&& git pull origin master --rebase && git push && cd - && rm -rf %s" \
% (self.obs_pkg_rpms_files_dir, self.obs_pkg_rpms_file, \
self.obs_pkg_rpms_file, self.obs_pkg_rpms_files_dir)
if os.system(cmd) == 0:
break
def update_repos_db(self):
"""
update obs repos db
"""
cmd = "chown -R obsrun:obsrun %s/%s/%s/:full; obs_admin --rescan-repository %s %s %s" \
% (self.obs_project_root_path, self.rpms_to_repo_path, self.arch, \
self.rpms_to_repo_path.split("/")[0], self.rpms_to_repo_path.split("/")[1], self.arch)
log.debug(cmd)
ret = self.pex.ssh_cmd(cmd)
class GETDate(object):
"""
get the latest date to obs_pkg_rpms
"""
def __init__(self, **kwargs):
"""
kwargs: dict, init dict by 'a': 'A' style
The dict key_value as the following:
branch: which you wangt to get the package date from
source_server_user: The user name for you use in server
source_server_pwd: The password for your ip
source_server_port: The port for your ip
giteeuser: The gitee account
giteeuserpwd: The gitee passwd
"""
self.branch = kwargs["branch"]
self.cmd = Pexpect(kwargs["source_server_user"],
kwargs["source_server_ip"],
kwargs["source_server_pwd"],
kwargs["source_server_port"])
self.giteeuser = kwargs["gitee_user"]
self.giteeuserpwd = kwargs["gitee_pwd"]
self.sourcedir = "/srv/cache/obs/tar_scm/repo/next/"
self.pkg_time = {}
self.realdir = None
def _get_realdir(self):
"""
get the path where the branch in
"""
if self.branch == "master":
realbranch = "openEuler"
else:
realbranch = self.branch
branchdir = os.path.join(self.sourcedir, realbranch)
self.realdir = branchdir
return branchdir
def _get_branch_rpmlist(self, realdir):
"""
get the rpmlist for corresponding branch
"""
rpmlist = []
log.info("rpmlist_dir: %s" % realdir)
cmd = "ls %s" % realdir
cmd_result = self.cmd.ssh_cmd(cmd)
log.info("ssh_cmd_for_ls: %s" % cmd_result)
for rpm in cmd_result:
if rpm.decode("utf-8").replace('\r\n', '') != ": ":
rpmlist.append(rpm.decode("utf-8").replace('\r\n', ''))
log.info("rpmlist: %s", ",".join(rpmlist))
return rpmlist
def _get_two_digis(self, int_num):
"""
make one digit into two digits
"""
if int_num < 10:
str_num = "0" + str(int_num)
else:
str_num = str(int_num)
return str_num
def _get_latest_time(self, rpm):
"""
get a fixed format date
rpm: which package you want to get
"""
cmd = "show -s --format=%ad --date=local"
pkg_dir = os.path.join(self.realdir, rpm)
show_result = self.cmd.ssh_cmd("git -C %s %s" % (pkg_dir, cmd))
log.info("%s : %s" % (rpm, show_result[1].decode("utf-8").replace('\r\n', '')))
str_day = show_result[1].decode("utf-8").replace('\r\n', '').split()[2]
year = show_result[1].decode("utf-8").replace('\r\n', '').split()[4]
timestr = show_result[1].decode("utf-8").replace('\r\n', '').split()[3]
eng_month = show_result[1].decode("utf-8").replace('\r\n', '').split()[1]
int_month = list(calendar.month_abbr).index(eng_month)
day = self._get_two_digis(int(str_day))
month = self._get_two_digis(int_month)
log.info("year:%s month:%s day:%s timestr:%s" % (year, month, day, timestr))
datestr = year + month + day + ' ' + timestr.replace(':', '-')
log.info(datestr)
self.pkg_time[rpm] = datestr
def _get_all_time_thread(self):
"""
use the thread pool to get the time
"""
realdir = self._get_realdir()
rpmlist = self._get_branch_rpmlist(realdir)
with ThreadPoolExecutor(10) as executor:
for pkg in rpmlist:
executor.submit(self._get_latest_time, pkg)
print(self.pkg_time)
def _echo_to_git_thread(self):
"""
use the thread pool to echo time str
"""
all_time = self.pkg_time
with ThreadPoolExecutor(10) as executor:
for rpm, time in all_time.items():
time_args = [rpm, time]
executor.submit(self._echo_to_obs_pkg_rpms, time_args)
def _clone_obs_pkg_rpms(self):
"""
clone the obs_pkg_rpms repository
"""
clone_path = os.getcwd()
log.info("Git clone the obs_pkg_rpms %s" % clone_path)
remove_git = os.popen("rm -rf ./obs_pkg_rpms").read()
log.info("rm result: %s" % remove_git)
repo_url = "https://%s:%[email protected]/unsunghero/obs_pkg_rpms" \
% (self.giteeuser, self.giteeuserpwd)
clone_cmd = "git clone --depth=1 %s" % repo_url
for i in range(5):
clone_result = subprocess.getstatusoutput(clone_cmd)
log.info(clone_result)
if clone_result[0] == 0:
clone_flag = "yes"
break
else:
rm_repo = os.popen("rm -rf obs_pkg_rpms").read()
if clone_flag != "yes":
raise SystemExit('Please check you internet!!!!!')
def _echo_to_obs_pkg_rpms(self, all_time):
"""
echo the time str to a package
all_time: the list for rpm name and timestr
"""
echo_cmd = "echo %s > ./obs_pkg_rpms/%s/%s" % (all_time[1], \
self.branch, all_time[0])
log.info(echo_cmd)
cmd_result = os.system(echo_cmd)
if cmd_result != 0:
raise SystemExit("This %s echo error to %s" % (all_time[0], self.branch))
def _push_to_pkg_rpms(self):
"""
push the latest obs_pkg_rpms to origin
"""
os.chdir("obs_pkg_rpms")
status_cmd = "git status -s"
commit_cmd = "git add -A && git commit -m 'update for package'"
if os.popen(status_cmd).read():
if os.system(commit_cmd) == 0:
for i in range(5):
push_cmd = "git push -f"
push_result = subprocess.getstatusoutput(push_cmd)
log.info(push_result)
if push_result[0] == 0:
log.info("SUCCESS:Push success for latest date to obs_pkg_rpms")
return
else:
log.debug("Try Push to obs_pkg_rpms: %s" % i)
raise SystemExit("Failed: Push to obs_pkg_rpms failed")
else:
log.info("NO CHAGE,nothing to commit")
def update_to_obs_pkg_rpms(self):
"""
Assemble all the processes that get the latest data
"""
self._clone_obs_pkg_rpms()
self._get_all_time_thread()
self._echo_to_git_thread()
self._push_to_pkg_rpms()