def load_from_url(self, url):
i = []
tmpdir = utils.mkdtemp(self.tmpbase)
testdef_file = download_image(url, self.context, tmpdir)
with open(testdef_file, 'r') as f:
logging.debug('loading test definition ...')
testdef = yaml.safe_load(f)
for e in INVALID_CHARS:
i.extend(indices(testdef["metadata"]["name"], e))
if i:
msg = "Test name contains invalid symbol(s) at position(s): %s" % ", ".join(map(str, i))
raise GeneralError(msg)
try:
testdef["metadata"]["name"].encode()
except UnicodeEncodeError as e:
msg = "Test name contains non-ascii symbols: %s" % e
raise GeneralError(msg)
if 'test-case-deps' in testdef:
self._get_dependent_test_cases(testdef)
idx = len(self.testdefs)
testdef_metadata = {'url': url, 'location': 'URL'}
testdef_metadata.update(_get_testdef_info(testdef))
self._append_testdef(URLTestDefinition(self.context, idx, testdef,
testdef_metadata))
def working_dir(self):
if self.config.shared_working_directory is None or \
self.config.shared_working_directory.strip() == '':
return self.scratch_dir
if self._working_dir is None:
self._working_dir = mkdtemp(self.config.shared_working_directory)
return self._working_dir
def working_dir(self):
if self.config.shared_working_directory is None or \
self.config.shared_working_directory.strip() == '':
return self.scratch_dir
if self._working_dir is None:
self._working_dir = mkdtemp(self.config.shared_working_directory)
return self._working_dir
def __init__(self, testdef_obj, handlers=None, device_config_vars=None):
if not device_config_vars:
device_config_vars = {}
if not handlers:
handlers = {}
SignalHandler.__init__(self, testdef_obj)
self.result_dir = utils.mkdtemp()
self.handlers = handlers
self.scratch_dir = utils.mkdtemp()
self.code_dir = os.path.join(self.scratch_dir, 'code')
shutil.copytree(testdef_obj.repo, self.code_dir)
device_config = testdef_obj.context.client.target_device.config
self.our_env = os.environ.copy()
for env_var, config_var in device_config_vars.iteritems():
try:
config_value = device_config.cp.get('__main__', config_var)
except NoOptionError:
logging.warning(
"No value found for device config %s; leaving %s unset "
"in environment", config_var, env_var)
else:
self.our_env[env_var] = config_value
def __init__(self, testdef_obj, handlers=None, device_config_vars=None):
if not device_config_vars:
device_config_vars = {}
if not handlers:
handlers = {}
SignalHandler.__init__(self, testdef_obj)
self.result_dir = utils.mkdtemp()
self.handlers = handlers
self.scratch_dir = utils.mkdtemp()
self.code_dir = os.path.join(self.scratch_dir, 'code')
shutil.copytree(testdef_obj.repo, self.code_dir)
device_config = testdef_obj.context.client.target_device.config
self.our_env = os.environ.copy()
for env_var, config_var in device_config_vars.iteritems():
try:
config_value = device_config.cp.get('__main__', config_var)
except NoOptionError:
logging.warning(
"No value found for device config %s; leaving %s unset "
"in environment", config_var, env_var)
else:
self.our_env[env_var] = config_value
def copy_to(self, context, local_path=None):
# copy file from image to local path
des_path = None
# make default local path in the lava
if local_path is None:
local_path = utils.mkdtemp(
basedir=context.config.lava_image_tmpdir)
src_path = '%s/%s' % (self.part_mntdir, self.path)
if not unicode_path_check(src_path):
raise CriticalError(
'Can not find source in image (%s at part %s)!' %
(self.path, self.part))
if os.path.isdir(src_path):
if not unicode_path_check(local_path):
des_path = local_path
else:
if self.file_name == '':
des_name = self.dir_name
else:
des_name = self.file_name
des_path = os.path.join(local_path, des_name)
logging.debug("Copying dir from #%s:%s(%s) to %s!", self.part,
self.path, src_path, des_path)
shutil.copytree(src_path, des_path)
elif os.path.isfile(src_path):
if not unicode_path_check(local_path):
if os.path.basename(local_path) == '':
des_name = os.path.basename(src_path)
des_path = os.path.join(local_path, des_name)
os.makedirs(local_path)
else:
if not unicode_path_check(os.path.dirname(local_path)):
os.makedirs(os.path.dirname(local_path))
des_path = local_path
else:
if os.path.isdir(local_path):
des_name = os.path.basename(src_path)
des_path = os.path.join(local_path, des_name)
else:
des_path = local_path
logging.debug("Copying file from #%s:%s(%s) to %s!", self.part,
self.path, src_path, des_path)
shutil.copyfile(src_path, des_path)
else:
raise CriticalError(
'Please check the source file type, we only support file and dir!'
)
return des_path
def _setup_tmpdir(self):
if not self.config.use_lava_tmpdir:
if self.config.alternative_dir is None:
logging.error("You have specified not to use the LAVA temporary \
directory. However, you have not defined an \
alternate temporary directory. Falling back to \
use the LAVA temporary directory.")
return self.context.config.lava_image_tmpdir, self.scratch_dir
else:
if self.config.alternative_create_tmpdir:
return self.config.alternative_dir, utils.mkdtemp(self.config.alternative_dir)
else:
return self.config.alternative_dir, self.config.alternative_dir
else:
return self.context.config.lava_image_tmpdir, self.scratch_dir
def _setup_tmpdir(self):
if not self.config.use_lava_tmpdir:
if self.config.alternative_dir is None:
logging.error("You have specified not to use the LAVA temporary \
directory. However, you have not defined an \
alternate temporary directory. Falling back to \
use the LAVA temporary directory.")
return self.context.config.lava_image_tmpdir, self.scratch_dir
else:
if self.config.alternative_create_tmpdir:
return self.config.alternative_dir, utils.mkdtemp(self.config.alternative_dir)
else:
return self.config.alternative_dir, self.config.alternative_dir
else:
return self.context.config.lava_image_tmpdir, self.scratch_dir
def load_from_url(self, url):
tmpdir = utils.mkdtemp(self.tmpbase)
testdef_file = download_image(url, self.context, tmpdir)
with open(testdef_file, 'r') as f:
logging.debug('loading test definition ...')
testdef = yaml.safe_load(f)
if 'test-case-deps' in testdef:
self._get_dependent_test_cases(testdef)
idx = len(self.testdefs)
testdef_metadata = {'url': url, 'location': 'URL'}
testdef_metadata.update(_get_testdef_info(testdef))
self._append_testdef(URLTestDefinition(self.context, idx, testdef,
testdef_metadata))
def copy_to(self, context, local_path=None):
# copy file from image to local path
des_path = None
# make default local path in the lava
if local_path is None:
local_path = utils.mkdtemp(basedir=context.config.lava_image_tmpdir)
src_path = '%s/%s' % (self.part_mntdir, self.path)
if not unicode_path_check(src_path):
raise CriticalError('Can not find source in image (%s at part %s)!' % (self.path, self.part))
if os.path.isdir(src_path):
if not unicode_path_check(local_path):
des_path = local_path
else:
if self.file_name == '':
des_name = self.dir_name
else:
des_name = self.file_name
des_path = os.path.join(local_path, des_name)
logging.debug("Copying dir from #%s:%s(%s) to %s!", self.part, self.path, src_path, des_path)
shutil.copytree(src_path, des_path)
elif os.path.isfile(src_path):
if not unicode_path_check(local_path):
if os.path.basename(local_path) == '':
des_name = os.path.basename(src_path)
des_path = os.path.join(local_path, des_name)
os.makedirs(local_path)
else:
if not unicode_path_check(os.path.dirname(local_path)):
os.makedirs(os.path.dirname(local_path))
des_path = local_path
else:
if os.path.isdir(local_path):
des_name = os.path.basename(src_path)
des_path = os.path.join(local_path, des_name)
else:
des_path = local_path
logging.debug("Copying file from #%s:%s(%s) to %s!", self.part, self.path, src_path, des_path)
shutil.copyfile(src_path, des_path)
else:
raise CriticalError('Please check the source file type, we only support file and dir!')
return des_path
def _get_git_repo(self, git_repo, branch, revision):
tmpdir = utils.mkdtemp()
logging.info('clone git-repo and change workspace to %s' % tmpdir)
os.chdir(tmpdir)
current_user = os.listdir('/home')[0]
logging.info('current user is: %s' % current_user)
cmd = ['sudo', '-u', current_user, 'git', 'clone']
if branch:
cmd.append('-b')
cmd.append(branch)
cmd.append(git_repo)
logging.info('cmd is: %s' % cmd)
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
name = os.path.splitext(os.path.basename(git_repo))[0]
logging.info('git_repo name is: %s' % name)
gitdir = os.path.join(tmpdir, name)
logging.info('git repo directory name is: %s' % git_repo)
if revision:
os.chdir(gitdir)
subprocess.check_output(['sudo', '-u', current_user, 'git', 'checkout', revision],
stderr=subprocess.STDOUT)
git_info = self._git_info(gitdir, name, current_user)
logging.info('git info: %s' % git_info)
return gitdir, git_info
def _read_boot_cmds(self, image=None, boot_tgz=None):
boot_file_path = None
if not self.config.read_boot_cmds_from_image:
return
# If we have already obtained boot commands dynamically, then return.
if self.__boot_cmds_dynamic__ is not None:
logging.debug("We already have boot commands in place.")
return
if image:
boot_part = self.config.boot_part
# Read boot related file from the boot partition of image.
with image_partition_mounted(image, boot_part) as mnt:
for boot_file in self.config.boot_files:
boot_path = os.path.join(mnt, boot_file)
if os.path.exists(boot_path):
boot_file_path = boot_path
break
elif boot_tgz:
tmp_dir = mkdtemp()
extracted_files = extract_tar(boot_tgz, tmp_dir)
for boot_file in self.config.boot_files:
for file_path in extracted_files:
if boot_file == os.path.basename(file_path):
boot_file_path = file_path
break
if boot_file_path and os.path.exists(boot_file_path):
with open(boot_file_path, 'r') as f:
boot_cmds = self._rewrite_boot_cmds(f.read())
self.__boot_cmds_dynamic__ = boot_cmds
else:
logging.debug("Unable to read boot commands dynamically.")
def _read_boot_cmds(self, image=None, boot_tgz=None):
boot_file_path = None
if not self.config.read_boot_cmds_from_image:
return
# If we have already obtained boot commands dynamically, then return.
if self.__boot_cmds_dynamic__ is not None:
logging.debug("We already have boot commands in place.")
return
if image:
boot_part = self.config.boot_part
# Read boot related file from the boot partition of image.
with image_partition_mounted(image, boot_part) as mnt:
for boot_file in self.config.boot_files:
boot_path = os.path.join(mnt, boot_file)
if os.path.exists(boot_path):
boot_file_path = boot_path
break
elif boot_tgz:
tmp_dir = mkdtemp()
extracted_files = extract_tar(boot_tgz, tmp_dir)
for boot_file in self.config.boot_files:
for file_path in extracted_files:
if boot_file == os.path.basename(file_path):
boot_file_path = file_path
break
if boot_file_path and os.path.exists(boot_file_path):
with open(boot_file_path, 'r') as f:
boot_cmds = self._rewrite_boot_cmds(f.read())
self.__boot_cmds_dynamic__ = boot_cmds
else:
logging.debug("Unable to read boot commands dynamically.")
def deploy_linaro_kernel(
self,
kernel,
ramdisk,
dtb,
overlays,
rootfs,
nfsrootfs,
image,
bootloader,
firmware,
bl0,
bl1,
bl2,
bl31,
rootfstype,
bootloadertype,
target_type,
scratch_dir,
qemu_pflash=None,
):
kernel_url = kernel
dtb_url = dtb
# At a minimum we must have a kernel
if kernel is None:
raise CriticalError("No kernel image to boot")
# Set the server IP (Dispatcher)
self._boot_tags["{SERVER_IP}"] = self.context.config.lava_server_ip
# Get the JTAG command fragments
stmc_command = " ".join([self.config.jtag_stmc_boot_script, self.config.jtag_stmc_boot_options])
# We have been passed kernel image
kernel = download_image(kernel, self.context, scratch_dir, decompress=False)
stmc_command = " ".join([stmc_command, self.config.jtag_stmc_kernel_command.format(KERNEL=kernel)])
if ramdisk is not None:
# We have been passed a ramdisk
ramdisk = download_image(ramdisk, self.context, scratch_dir, decompress=False)
if overlays is not None:
ramdisk_dir = extract_ramdisk(ramdisk, scratch_dir, is_uboot=False)
for overlay in overlays:
overlay = download_image(overlay, self.context, scratch_dir, decompress=False)
extract_overlay(overlay, ramdisk_dir)
ramdisk = create_ramdisk(ramdisk_dir, scratch_dir)
stmc_command = " ".join([stmc_command, self.config.jtag_stmc_ramdisk_command.format(RAMDISK=ramdisk)])
if dtb is not None:
# We have been passed a device tree blob
dtb = download_image(dtb, self.context, scratch_dir, decompress=False)
stmc_command = " ".join([stmc_command, self.config.jtag_stmc_dtb_command.format(DTB=dtb)])
if nfsrootfs is not None:
# Extract rootfs into nfsrootfs directory
nfsrootfs = download_image(nfsrootfs, self.context, scratch_dir, decompress=False)
scratch_dir = mkdtemp(self.context.config.lava_image_tmpdir)
lava_nfsrootfs = mkdtemp(basedir=scratch_dir)
extract_rootfs(nfsrootfs, lava_nfsrootfs)
self._boot_tags["{NFSROOTFS}"] = lava_nfsrootfs
self._default_boot_cmds = "boot_cmds_nfs"
if overlays is not None and ramdisk is None:
for overlay in overlays:
overlay = download_image(overlay, self.context, scratch_dir, decompress=False)
extract_overlay(overlay, lava_nfsrootfs)
# Add suffix for boot commands
self._stmc_command = stmc_command + " --"
if self.context.test_data.metadata.get("is_slave", "false") == "true":
logging.info("Booting in the master/slave mode, as *slave*")
logging.info("Sending the kernel, dtb, nfsrootfs urls")
self.context.transport.request_send(
"lava_ms_slave_data",
{
"kernel": kernel_url,
"dtb": dtb_url if dtb_url else "",
"nfs_rootfs": lava_nfsrootfs,
"nfs_server_ip": self.context.config.lava_server_ip,
"stmc_ip": self.config.jtag_stmc_ip,
},
)
return self._boot_tags, self._default_boot_cmds
def get_www_scratch_dir(self):
""" returns a temporary directory available for downloads that gets
deleted when the process exits """
return mkdtemp(self.context.config.lava_image_tmpdir)
def _setup_nfs(self, nfsrootfs, tmpdir):
lava_nfsrootfs = utils.mkdtemp(basedir=tmpdir)
utils.extract_rootfs(nfsrootfs, lava_nfsrootfs)
return lava_nfsrootfs
def scratch_dir(self):
if self._scratch_dir is None:
self._scratch_dir = utils.mkdtemp(
self.context.config.lava_image_tmpdir)
return self._scratch_dir
def load_from_repo(self, testdef_repo):
tmpdir = utils.mkdtemp(self.tmpbase)
repo = None
info = None
if 'git-repo' in testdef_repo:
repo = _get_testdef_git_repo(testdef_repo['git-repo'], tmpdir,
testdef_repo.get('revision'),
_get_lava_proxy(self.context))
name = os.path.splitext(os.path.basename(testdef_repo['git-repo']))[0]
info = _git_info(testdef_repo['git-repo'], repo, name)
if 'bzr-repo' in testdef_repo:
repo = _get_testdef_bzr_repo(testdef_repo['bzr-repo'], tmpdir,
testdef_repo.get('revision'),
_get_lava_proxy(self.context))
name = testdef_repo['bzr-repo'].replace('lp:', '').split('/')[-1]
info = _bzr_info(testdef_repo['bzr-repo'], repo, name)
if 'tar-repo' in testdef_repo:
repo = _get_testdef_tar_repo(testdef_repo['tar-repo'], tmpdir)
# Default info structure, since we need something, but we have
# a tar file in this case.
info = {
"project_name": "Tar archived repository",
"branch_vcs": "tar",
"branch_revision": "0",
"branch_url": repo
}
if 'url' in testdef_repo:
repo = _get_testdef_url_repo(testdef_repo['url'], self.context, tmpdir)
testdef_repo['testdef'] = os.path.basename(testdef_repo['url'])
# Default info structure. Due to branch_vcs is enum type,
# we use tar temporarily. For Dashboard Bundle Format 1.6,
# "enum": ["bzr", "git", "svn", "tar"]
info = {
"project_name": "URL repository",
"branch_vcs": "url",
"branch_revision": "0",
"branch_url": repo
}
if not repo or not info:
logging.warning("Unable to identify specified repository. %s" %
testdef_repo)
else:
if 'parameters' in testdef_repo:
# get the parameters for test.
logging.debug('Get test parameters : %s' % testdef_repo['parameters'])
info['test_params'] = str(testdef_repo['parameters'])
else:
info['test_params'] = ''
test = testdef_repo.get('testdef', 'lavatest.yaml')
try:
with open(os.path.join(repo, test), 'r') as f:
logging.debug('loading test definition ...')
testdef = yaml.safe_load(f)
except IOError as e:
msg = "Unable to load test definition '%s/%s': %s" % (os.path.basename(repo), test, e)
raise CriticalError(msg)
if 'test-case-deps' in testdef:
self._get_dependent_test_cases(testdef)
# for test paramters
if 'params' in testdef:
logging.debug('Get default parameters : %s' % testdef['params'])
info['default_params'] = str(testdef['params'])
else:
info['default_params'] = ''
idx = len(self.testdefs)
self._append_testdef(
RepoTestDefinition(self.context, idx, testdef, repo, info))
def scratch_dir(self):
if self._scratch_dir is None:
self._scratch_dir = utils.mkdtemp(
self.context.config.lava_image_tmpdir)
return self._scratch_dir
def get_www_scratch_dir(self):
""" returns a temporary directory available for downloads that gets
deleted when the process exits """
return mkdtemp(self.context.config.lava_image_tmpdir)
def deploy_linaro_kernel(self, kernel, ramdisk, dtb, modules, rootfs, nfsrootfs,
bootloader, firmware, bl1, bl2, bl31, rootfstype,
bootloadertype, target_type, scratch_dir):
kernel_url = kernel
# At a minimum we must have a kernel
if kernel is None:
raise CriticalError("No kernel image to boot")
# Set the server IP (Dispatcher)
self._boot_tags['{SERVER_IP}'] = self.context.config.lava_server_ip
# Get the JTAG command fragments
stmc_command = ' '.join([self.config.jtag_stmc_boot_script,
self.config.jtag_stmc_boot_options])
# We have been passed kernel image
kernel = download_image(kernel, self.context,
scratch_dir, decompress=False)
stmc_command = ' '.join([stmc_command,
self.config.jtag_stmc_kernel_command.format(KERNEL=kernel)])
if ramdisk is not None:
# We have been passed a ramdisk
ramdisk = download_image(ramdisk, self.context,
scratch_dir,
decompress=False)
if modules is not None:
modules = download_image(modules, self.context,
scratch_dir,
decompress=False)
ramdisk_dir = extract_ramdisk(ramdisk, scratch_dir,
is_uboot=False)
extract_modules(modules, ramdisk_dir)
ramdisk = create_ramdisk(ramdisk_dir, scratch_dir)
stmc_command = ' '.join([stmc_command,
self.config.jtag_stmc_ramdisk_command.format(RAMDISK=ramdisk)])
if dtb is not None:
# We have been passed a device tree blob
dtb = download_image(dtb, self.context,
scratch_dir, decompress=False)
stmc_command = ' '.join([stmc_command,
self.config.jtag_stmc_dtb_command.format(DTB=dtb)])
if nfsrootfs is not None:
# Extract rootfs into nfsrootfs directory
nfsrootfs = download_image(nfsrootfs, self.context,
scratch_dir,
decompress=False)
scratch_dir = mkdtemp(self.context.config.lava_image_tmpdir)
lava_nfsrootfs = mkdtemp(basedir=scratch_dir)
extract_rootfs(nfsrootfs, lava_nfsrootfs)
self._boot_tags['{NFSROOTFS}'] = lava_nfsrootfs
self._default_boot_cmds = 'boot_cmds_nfs'
if modules is not None and ramdisk is None:
modules = download_image(modules, self.context,
scratch_dir,
decompress=False)
extract_modules(modules, lava_nfsrootfs)
# Add suffix for boot commands
self._stmc_command = stmc_command + ' --'
if self.context.test_data.metadata.get('is_slave', 'false') == 'true':
logging.info("Booting in the master/slave mode, as *slave*")
logging.info("Sending the kernel, dtb, nfsrootfs urls")
self.context.transport.request_send('lava_ms_slave_data',
{'kernel': kernel_url,
'nfs_rootfs': lava_nfsrootfs,
'nfs_server_ip': self.context.config.lava_server_ip,
'stmc_ip': self.config.jtag_stmc_ip})
return self._boot_tags, self._default_boot_cmds
def deploy_linaro_kernel(self, kernel, ramdisk, dtb, overlays, rootfs, nfsrootfs,
image, bootloader, firmware, bl0, bl1, bl2, bl31, rootfstype,
bootloadertype, target_type, scratch_dir, qemu_pflash=None):
kernel_url = kernel
dtb_url = dtb
# At a minimum we must have a kernel
if kernel is None:
raise CriticalError("No kernel image to boot")
# Set the server IP (Dispatcher)
self._boot_tags['{SERVER_IP}'] = self.context.config.lava_server_ip
# Get the JTAG command fragments
stmc_command = ' '.join([self.config.jtag_stmc_boot_script,
self.config.jtag_stmc_boot_options])
# We have been passed kernel image
kernel = download_image(kernel, self.context,
scratch_dir, decompress=False)
stmc_command = ' '.join([stmc_command,
self.config.jtag_stmc_kernel_command.format(KERNEL=kernel)])
if ramdisk is not None:
# We have been passed a ramdisk
ramdisk = download_image(ramdisk, self.context,
scratch_dir,
decompress=False)
if overlays is not None:
ramdisk_dir = extract_ramdisk(ramdisk, scratch_dir,
is_uboot=False)
for overlay in overlays:
overlay = download_image(overlay, self.context,
scratch_dir,
decompress=False)
extract_overlay(overlay, ramdisk_dir)
ramdisk = create_ramdisk(ramdisk_dir, scratch_dir)
stmc_command = ' '.join([stmc_command,
self.config.jtag_stmc_ramdisk_command.format(RAMDISK=ramdisk)])
if dtb is not None:
# We have been passed a device tree blob
dtb = download_image(dtb, self.context,
scratch_dir, decompress=False)
stmc_command = ' '.join([stmc_command,
self.config.jtag_stmc_dtb_command.format(DTB=dtb)])
if nfsrootfs is not None:
# Extract rootfs into nfsrootfs directory
nfsrootfs = download_image(nfsrootfs, self.context,
scratch_dir,
decompress=False)
scratch_dir = mkdtemp(self.context.config.lava_image_tmpdir)
lava_nfsrootfs = mkdtemp(basedir=scratch_dir)
extract_rootfs(nfsrootfs, lava_nfsrootfs)
self._boot_tags['{NFSROOTFS}'] = lava_nfsrootfs
self._default_boot_cmds = 'boot_cmds_nfs'
if overlays is not None and ramdisk is None:
for overlay in overlays:
overlay = download_image(overlay, self.context,
scratch_dir,
decompress=False)
extract_overlay(overlay, lava_nfsrootfs)
# Add suffix for boot commands
self._stmc_command = stmc_command + ' --'
if self.context.test_data.metadata.get('is_slave', 'false') == 'true':
logging.info("Booting in the master/slave mode, as *slave*")
logging.info("Sending the kernel, dtb, nfsrootfs urls")
self.context.transport.request_send('lava_ms_slave_data',
{'kernel': kernel_url,
'dtb': dtb_url if dtb_url else '',
'nfs_rootfs': lava_nfsrootfs,
'nfs_server_ip': self.context.config.lava_server_ip,
'stmc_ip': self.config.jtag_stmc_ip})
return self._boot_tags, self._default_boot_cmds
def _setup_nfs(self, nfsrootfs, tmpdir):
lava_nfsrootfs = utils.mkdtemp(basedir=tmpdir)
utils.extract_rootfs(nfsrootfs, lava_nfsrootfs)
return lava_nfsrootfs