def prepare_repos(self):
"""
Prepare repos for the tests.
"""
def merge_pulls(repo_name, pull_nos):
branch_name = ','.join(pull_nos)
cmd = 'git checkout -b %s' % branch_name
res = utils.run(cmd, ignore_status=True)
if res.exit_status:
print res
raise Exception('Failed to create branch %s' % branch_name)
for pull_no in pull_nos:
patch_url = ('https://github.com/autotest'
'/%s/pull/%s.patch' % (repo_name, pull_no))
patch_file = "/tmp/%s.patch" % pull_no
urllib.urlretrieve(patch_url, patch_file)
try:
cmd = 'git am -3 %s' % patch_file
res = utils.run(cmd, ignore_status=True)
except:
print res
raise Exception('Failed applying patch %s' % pull_no)
finally:
os.remove(patch_file)
return branch_name
def file_changed(repo_name):
cmd = 'git diff master --name-only'
res = utils.run(cmd, ignore_status=True)
if res.exit_status:
print res
raise Exception("Failed to get diff info against master")
return res.stdout.strip().splitlines()
self.virt_branch_name, self.libvirt_branch_name = None, None
if self.args.virt_test_pull:
os.chdir(data_dir.get_root_dir())
self.virt_branch_name = merge_pulls(
"virt-test",
self.args.virt_test_pull.split(','))
self.virt_file_changed = file_changed("virt-test")
if self.args.libvirt_pull:
os.chdir(data_dir.get_test_provider_dir(
'io-github-autotest-libvirt'))
self.libvirt_branch_name = merge_pulls(
"tp-libvirt",
self.args.libvirt_pull.split(','))
self.libvirt_file_changed = file_changed("tp-libvirt")
os.chdir(data_dir.get_root_dir())
def test_providers_not_downloaded():
"""
Return the list of test providers that have not being downloaded
"""
result = []
for provider in get_test_provider_names():
download_dst = data_dir.get_test_provider_dir(provider)
repo_downloaded = os.path.isdir(os.path.join(download_dst, '.git'))
if not repo_downloaded:
result.append(provider)
return result
def test_providers_not_downloaded():
"""
Return the list of test providers that have not being downloaded
"""
result = []
for provider in get_test_provider_names():
download_dst = data_dir.get_test_provider_dir(provider)
repo_downloaded = os.path.isdir(os.path.join(download_dst, '.git'))
if not repo_downloaded:
result.append(provider)
return result
def download_test_provider(provider, update=False):
"""
Download a test provider defined on a .ini file inside test-providers.d.
This function will only download test providers that are in git repos.
Local filesystems don't need this functionality.
:param provider: Test provider name, such as 'io-github-autotest-qemu'.
"""
provider_info = get_test_provider_info(provider)
uri = provider_info.get('uri')
if not uri.startswith('file://'):
uri = provider_info.get('uri')
branch = provider_info.get('branch')
ref = provider_info.get('ref')
pubkey = provider_info.get('pubkey')
download_dst = data_dir.get_test_provider_dir(provider)
dir_existed = os.path.exists(download_dst)
repo_downloaded = os.path.isdir(os.path.join(download_dst, '.git'))
original_dir = os.getcwd()
try:
if not repo_downloaded or update:
download_dst = git.get_repo(uri=uri,
branch=branch,
commit=ref,
destination_dir=download_dst)
os.chdir(download_dst)
try:
process.run('git remote add origin %s' % uri)
except process.CmdError:
pass
except Exception:
if not dir_existed and os.path.isdir(download_dst):
logging.error('Cleaning up provider %s download dir %s',
provider, download_dst)
shutil.rmtree(download_dst)
raise
# sanity check to ensure the git repository is OK:
try:
os.chdir(download_dst)
process.system('git log -1')
except process.CmdError:
logging.error(
'Something is unexpectedly wrong with the git repository at %s',
download_dst)
raise
finally:
os.chdir(original_dir)
def download_test_provider(provider, update=False):
"""
Download a test provider defined on a .ini file inside test-providers.d.
This function will only download test providers that are in git repos.
Local filesystems don't need this functionality.
:param provider: Test provider name, such as 'io-github-autotest-qemu'.
"""
provider_info = get_test_provider_info(provider)
uri = provider_info.get('uri')
if not uri.startswith('file://'):
uri = provider_info.get('uri')
branch = provider_info.get('branch')
ref = provider_info.get('ref')
pubkey = provider_info.get('pubkey')
download_dst = data_dir.get_test_provider_dir(provider)
dir_existed = os.path.exists(download_dst)
repo_downloaded = os.path.isdir(os.path.join(download_dst, '.git'))
original_dir = os.getcwd()
try:
if not repo_downloaded or update:
download_dst = git.get_repo(uri=uri, branch=branch, commit=ref,
destination_dir=download_dst)
os.chdir(download_dst)
try:
process.run('git remote add origin %s' % uri)
except process.CmdError:
pass
except Exception:
if not dir_existed and os.path.isdir(download_dst):
logging.error('Cleaning up provider %s download dir %s', provider,
download_dst)
shutil.rmtree(download_dst)
raise
# sanity check to ensure the git repository is OK:
try:
os.chdir(download_dst)
process.system('git log -1')
except process.CmdError:
logging.error('Something is unexpectedly wrong with the git repository at %s',
download_dst)
raise
finally:
os.chdir(original_dir)
def get_test_provider_info(provider):
"""
Get a dictionary with relevant test provider info, such as:
* provider uri (git repo or filesystem location)
* provider git repo data, such as branch, ref, pubkey
* backends that this provider has tests for. For each backend type the
provider has tests for, the 'path' will be also available.
:param provider: Test provider name, such as 'io-github-autotest-qemu'.
"""
provider_info = {}
provider_path = os.path.join(data_dir.get_test_providers_dir(),
'%s.ini' % provider)
provider_cfg = ConfigLoader(provider_path)
provider_info['name'] = provider
provider_info['uri'] = provider_cfg.get('provider', 'uri')
provider_info['branch'] = provider_cfg.get('provider', 'branch', 'master')
provider_info['ref'] = provider_cfg.get('provider', 'ref')
provider_info['pubkey'] = provider_cfg.get('provider', 'pubkey')
provider_info['backends'] = {}
for backend in get_known_backends():
subdir = provider_cfg.get(backend, 'subdir')
cart_cfgs = provider_cfg.get(backend, 'configs')
backend_dic = {}
if cart_cfgs is not None:
# Give ability to specify few required configs separated with comma
cart_cfgs = [x.strip() for x in cart_cfgs.split(',')]
backend_dic.update({'cartesian_configs': cart_cfgs})
if subdir is not None:
if provider_info['uri'].startswith('file://'):
src = os.path.join(provider_info['uri'][7:],
subdir)
else:
src = os.path.join(data_dir.get_test_provider_dir(provider),
subdir)
backend_dic.update({'path': src})
provider_info['backends'].update({backend: backend_dic})
return provider_info
def get_test_provider_info(provider):
"""
Get a dictionary with relevant test provider info, such as:
* provider uri (git repo or filesystem location)
* provider git repo data, such as branch, ref, pubkey
* backends that this provider has tests for. For each backend type the
provider has tests for, the 'path' will be also available.
:param provider: Test provider name, such as 'io-github-autotest-qemu'.
"""
provider_info = {}
provider_path = os.path.join(data_dir.get_test_providers_dir(),
'%s.ini' % provider)
provider_cfg = ConfigLoader(provider_path)
provider_info['name'] = provider
provider_info['uri'] = provider_cfg.get('provider', 'uri')
provider_info['branch'] = provider_cfg.get('provider', 'branch', 'master')
provider_info['ref'] = provider_cfg.get('provider', 'ref')
provider_info['pubkey'] = provider_cfg.get('provider', 'pubkey')
provider_info['backends'] = {}
for backend in get_known_backends():
subdir = provider_cfg.get(backend, 'subdir')
cart_cfgs = provider_cfg.get(backend, 'configs')
backend_dic = {}
if cart_cfgs is not None:
# Give ability to specify few required configs separated with comma
cart_cfgs = [x.strip() for x in cart_cfgs.split(',')]
backend_dic.update({'cartesian_configs': cart_cfgs})
if subdir is not None:
if provider_info['uri'].startswith('file://'):
src = os.path.join(provider_info['uri'][7:], subdir)
else:
src = os.path.join(data_dir.get_test_provider_dir(provider),
subdir)
backend_dic.update({'path': src})
provider_info['backends'].update({backend: backend_dic})
return provider_info
def change_to_only(change_list):
"""
Transform the content of a change file to a only set.
"""
onlys = set()
for line in change_list:
filename = line.strip()
res = re.match('libvirt/tests/(cfg|src)/(.*).(cfg|py)',
filename)
if res:
cfg_path = 'libvirt/tests/cfg/%s.cfg' % res.groups()[1]
tp_dir = data_dir.get_test_provider_dir(
'io-github-autotest-libvirt')
cfg_path = os.path.join(tp_dir, cfg_path)
try:
with open(cfg_path) as fcfg:
only = fcfg.readline().strip()
only = only.lstrip('-').rstrip(':').strip()
onlys.add(only)
except:
pass
return onlys
def restore_repos(self):
"""
Checkout master branch and remove test branch.
"""
def restore_repo(branch_name):
cmd = 'git checkout master'
res = utils.run(cmd, ignore_status=True)
if res.exit_status:
print res
cmd = 'git branch -D %s' % branch_name
res = utils.run(cmd, ignore_status=True)
if res.exit_status:
print res
if self.virt_branch_name:
os.chdir(data_dir.get_root_dir())
restore_repo(self.virt_branch_name)
if self.libvirt_branch_name:
os.chdir(data_dir.get_test_provider_dir(
'io-github-autotest-libvirt'))
restore_repo(self.libvirt_branch_name)
os.chdir(data_dir.get_root_dir())
