def check_iso(url, destination, hash):
"""
Verifies if ISO that can be find on url is on destination with right hash.
This function will verify the SHA1 hash of the ISO image. If the file
turns out to be missing or corrupted, let the user know we can download it.
@param url: URL where the ISO file can be found.
@param destination: Directory in local disk where we'd like the iso to be.
@param hash: SHA1 hash for the ISO image.
"""
logging.info("Verifying iso %s", os.path.basename(url))
if not destination:
os.makedirs(destination)
iso_path = os.path.join(destination, os.path.basename(url))
if not os.path.isfile(iso_path) or (
utils.hash_file(iso_path, method="sha1") != hash):
logging.warning("%s not found or corrupted", iso_path)
logging.warning("Would you like to download it? (y/n)")
iso_download = raw_input()
if iso_download == 'y':
utils.unmap_url_cache(destination, url, hash, method="sha1")
else:
logging.warning("Missing file %s. Please download it", iso_path)
else:
logging.debug("%s present, with proper checksum", iso_path)
def check_iso(url, destination, hash):
"""
Verifies if ISO that can be find on url is on destination with right hash.
This function will verify the SHA1 hash of the ISO image. If the file
turns out to be missing or corrupted, let the user know we can download it.
@param url: URL where the ISO file can be found.
@param destination: Directory in local disk where we'd like the iso to be.
@param hash: SHA1 hash for the ISO image.
"""
file_ok = False
if not destination:
os.makedirs(destination)
iso_path = os.path.join(destination, os.path.basename(url))
if not os.path.isfile(iso_path):
logging.warning("File %s not found", iso_path)
logging.warning("Expected SHA1 sum: %s", hash)
answer = utils.ask("Would you like to download it from %s?" % url)
if answer == 'y':
try:
utils.unmap_url_cache(destination, url, hash, method="sha1")
file_ok = True
except EnvironmentError, e:
logging.error(e)
else:
logging.warning("Missing file %s", iso_path)
logging.warning("Please download it or put an exsiting copy on the "
"appropriate location")
return
def check_iso(url, destination, hash):
"""
Verifies if ISO that can be find on url is on destination with right hash.
This function will verify the SHA1 hash of the ISO image. If the file
turns out to be missing or corrupted, let the user know we can download it.
@param url: URL where the ISO file can be found.
@param destination: Directory in local disk where we'd like the iso to be.
@param hash: SHA1 hash for the ISO image.
"""
file_ok = False
if not destination:
os.makedirs(destination)
iso_path = os.path.join(destination, os.path.basename(url))
if not os.path.isfile(iso_path):
logging.warning("File %s not found", iso_path)
logging.warning("Expected SHA1 sum: %s", hash)
answer = utils.ask("Would you like to download it from %s?" % url)
if answer == 'y':
try:
utils.unmap_url_cache(destination, url, hash, method="sha1")
file_ok = True
except EnvironmentError, e:
logging.error(e)
else:
logging.warning("Missing file %s", iso_path)
logging.warning(
"Please download it or put an exsiting copy on the "
"appropriate location")
return
def check_iso(url, destination, hash):
"""
Verifies if ISO that can be find on url is on destination with right hash.
This function will verify the SHA1 hash of the ISO image. If the file
turns out to be missing or corrupted, let the user know we can download it.
@param url: URL where the ISO file can be found.
@param destination: Directory in local disk where we'd like the iso to be.
@param hash: SHA1 hash for the ISO image.
"""
logging.info("Verifying iso %s", os.path.basename(url))
if not destination:
os.makedirs(destination)
iso_path = os.path.join(destination, os.path.basename(url))
if not os.path.isfile(iso_path) or (utils.hash_file(
iso_path, method="sha1") != hash):
logging.warning("%s not found or corrupted", iso_path)
logging.warning("Would you like to download it? (y/n)")
iso_download = raw_input()
if iso_download == 'y':
utils.unmap_url_cache(destination, url, hash, method="sha1")
else:
logging.warning("Missing file %s. Please download it", iso_path)
else:
logging.debug("%s present, with proper checksum", iso_path)
def run_once(self, test='antlr', config='./dacapo.cfg', jvm='default'):
cfg = config_loader(cfg=config, tmpdir=self.tmpdir, raise_errors=True)
self.test = test
cachedir = os.path.join(self.bindir, 'cache')
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
dacapo_url = cfg.get('dacapo', 'tarball_url')
dacapo_md5 = cfg.get('dacapo', 'package_md5')
dacapo_pkg = utils.unmap_url_cache(cachedir, dacapo_url, dacapo_md5)
if not jvm == 'default':
# Get the jvm package
jvm_pkg_url = cfg.get(jvm, 'jvm_pkg_url')
jvm_pkg_md5 = cfg.get(jvm, 'package_md5')
jvm_pkg = utils.unmap_url_cache(cachedir, jvm_pkg_url, jvm_pkg_md5)
# Install it
package.install(jvm_pkg)
# Basic Java environment variables setup
java_root = cfg.get(jvm, 'java_root')
self.set_java_environment(jvm, java_root)
if cfg.get('global', 'use_global') == 'yes':
iterations = cfg.get('global', 'iterations')
workload = cfg.get('global', 'workload')
else:
iterations = cfg.get(test, 'iterations')
workload = cfg.get(test, 'workload')
verbose = '-v '
workload = '-s %s ' % workload
iterations = '-n %s ' % iterations
self.scratch = os.path.join(self.resultsdir, test)
scratch = '--scratch-directory %s ' % self.scratch
args = verbose + workload + scratch + iterations + test
self.raw_result_file = os.path.join(self.resultsdir,
'raw_output_%s' % self.iteration)
raw_result = open(self.raw_result_file, 'w')
logging.info('Running dacapo benchmark %s', test)
try:
cmd = 'java -jar %s %s' % (dacapo_pkg, args)
results = utils.run(command=cmd, stdout_tee=raw_result,
stderr_tee=raw_result)
self.results = results.stderr
raw_result.close()
except error.CmdError, e:
raise error.TestError('Dacapo benchmark %s has failed: %s' %
(test, e))
def setup(self, tarball_base='linux-2.6.18.tar.bz2', parallel=True):
"""
Downloads a copy of the linux kernel, calculate an estimated size of
the uncompressed tarball, use this value to calculate the number of
copies of the linux kernel that will be uncompressed.
@param tarball_base: Name of the kernel tarball location that will
be looked up on the kernel.org mirrors.
@param parallel: If we are going to uncompress the copies of the
kernel in parallel or not
"""
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
self.parallel = parallel
kernel_repo = 'http://www.kernel.org/pub/linux/kernel/v2.6'
tarball_url = os.path.join(kernel_repo, tarball_base)
tarball_md5 = '296a6d150d260144639c3664d127d174'
logging.info('Downloading linux kernel tarball')
self.tarball = utils.unmap_url_cache(self.cachedir, tarball_url,
tarball_md5)
size_tarball = os.path.getsize(self.tarball) / 1024 / 1024
# Estimation of the tarball size after uncompression
compress_ratio = 5
est_size = size_tarball * compress_ratio
self.sim_cps = self.get_sim_cps(est_size)
logging.info('Source file: %s' % tarball_base)
logging.info('Megabytes per copy: %s' % size_tarball)
logging.info('Compress ratio: %s' % compress_ratio)
logging.info('Estimated size after uncompression: %s' % est_size)
logging.info('Number of copies: %s' % self.sim_cps)
logging.info('Parallel: %s' % parallel)
def setup(self, tarball_base='linux-2.6.18.tar.bz2', parallel=True):
"""
Downloads a copy of the linux kernel, calculate an estimated size of
the uncompressed tarball, use this value to calculate the number of
copies of the linux kernel that will be uncompressed.
@param tarball_base: Name of the kernel tarball location that will
be looked up on the kernel.org mirrors.
@param parallel: If we are going to uncompress the copies of the
kernel in parallel or not
"""
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
self.parallel = parallel
kernel_repo = 'http://www.kernel.org/pub/linux/kernel/v2.6'
tarball_url = os.path.join(kernel_repo, tarball_base)
tarball_md5 = '296a6d150d260144639c3664d127d174'
logging.info('Downloading linux kernel tarball')
self.tarball = utils.unmap_url_cache(self.cachedir, tarball_url,
tarball_md5)
size_tarball = os.path.getsize(self.tarball) / 1024 / 1024
# Estimation of the tarball size after uncompression
compress_ratio = 5
est_size = size_tarball * compress_ratio
self.sim_cps = self.get_sim_cps(est_size)
logging.info('Source file: %s' % tarball_base)
logging.info('Megabytes per copy: %s' % size_tarball)
logging.info('Compress ratio: %s' % compress_ratio)
logging.info('Estimated size after uncompression: %s' % est_size)
logging.info('Number of copies: %s' % self.sim_cps)
logging.info('Parallel: %s' % parallel)
def install_lsb_packages(self):
if not self.packages_installed:
# First, we download the LSB DTK manager package, worry about
# installing it later
dtk_manager_arch = self.config.get('dtk-manager', 'arch-%s' % self.arch)
dtk_manager_url = self.config.get('dtk-manager',
'tarball_url') % dtk_manager_arch
if not dtk_manager_url:
raise error.TestError('Could not get DTK manager URL from'
' configuration file')
dtk_md5 = self.config.get('dtk-manager', 'md5-%s' % self.arch)
if dtk_md5:
logging.info('Caching LSB DTK manager RPM')
dtk_manager_pkg = utils.unmap_url_cache(self.cachedir,
dtk_manager_url,
dtk_md5)
else:
raise error.TestError('Could not find DTK manager package md5,'
' cannot cache DTK manager tarball')
# Get LSB tarball, cache it and uncompress under autotest srcdir
if self.config.get('lsb', 'override_default_url') == 'no':
lsb_url = self.config.get('lsb', 'tarball_url') % self.arch
else:
lsb_url = self.config.get('lsb', 'tarball_url_alt') % self.arch
if not lsb_url:
raise error.TestError('Could not get LSB URL from configuration'
' file')
md5_key = 'md5-%s' % self.arch
lsb_md5 = self.config.get('lsb', md5_key)
if lsb_md5:
logging.info('Caching LSB tarball')
lsb_pkg = utils.unmap_url_cache(self.cachedir, lsb_url, lsb_md5)
else:
raise error.TestError('Could not find LSB package md5, cannot'
' cache LSB tarball')
utils.extract_tarball_to_dir(lsb_pkg, self.srcdir)
# Lets load a file that contains the list of RPMs
os.chdir(self.srcdir)
if not os.path.isfile('inst-config'):
raise IOError('Could not find file with package info,'
' inst-config')
rpm_file_list = open('inst-config', 'r')
pkg_pattern = re.compile('[A-Za-z0-9_.-]*[.][r][p][m]')
lsb_pkg_list = []
for line in rpm_file_list.readlines():
try:
# We will install lsb-dtk-manager separately, so we can remove
# it from the list of packages
if not 'lsb-dtk-manager' in line:
line = re.findall(pkg_pattern, line)[0]
lsb_pkg_list.append(line)
except:
# If we don't get a match, no problem
pass
# Lets figure out the host distro
distro_pkg_support = package.os_support()
if os.path.isfile('/etc/debian_version') and \
distro_pkg_support['dpkg']:
logging.debug('Debian based distro detected')
if distro_pkg_support['conversion']:
logging.debug('Package conversion supported')
distro_type = 'debian-based'
else:
raise EnvironmentError('Package conversion not supported.'
'Cannot handle LSB package'
' installation')
elif distro_pkg_support['rpm']:
logging.debug('Red Hat based distro detected')
distro_type = 'redhat-based'
else:
logging.error('OS does not seem to be red hat or debian based')
raise EnvironmentError('Cannot handle LSB package installation')
# According to the host distro detection, we can install the packages
# using the list previously assembled
if distro_type == 'redhat-based':
logging.info('Installing LSB RPM packages')
package.install(dtk_manager_pkg)
for lsb_rpm in lsb_pkg_list:
package.install(lsb_rpm, nodeps=True)
elif distro_type == 'debian-based':
logging.info('Remember that you must have the following lsb'
' compliance packages installed:')
logging.info('lsb-core lsb-cxx lsb-graphics lsb-desktop lsb-qt4'
' lsb-languages lsb-multimedia lsb-printing')
logging.info('Converting and installing LSB packages')
dtk_manager_dpkg = package.convert(dtk_manager_pkg, 'dpkg')
package.install(dtk_manager_dpkg)
for lsb_rpm in lsb_pkg_list:
lsb_dpkg = package.convert(lsb_rpm, 'dpkg')
package.install(lsb_dpkg, nodeps=True)
self.packages_installed = True
def execute(self, test = 'antlr', config = './dacapo.cfg', jvm = 'ibm14-ppc64'):
# Load the test configuration. If needed, use autotest tmpdir to write
# files.
my_config = config_loader(config, self.tmpdir)
# Directory where we will cache the dacapo jar file
# and the jvm package files
self.cachedir = os.path.join(self.bindir, 'cache')
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
# Get dacapo jar URL
# (It's possible to override the default URL that points to the
# sourceforge repository)
if my_config.get('dacapo', 'override_default_url') == 'no':
self.dacapo_url = my_config.get('dacapo', 'tarball_url')
else:
self.dacapo_url = my_config.get('dacapo', 'tarball_url_alt')
if not self.dacapo_url:
raise error.TestError('Could not read dacapo URL from conf file')
# We can cache the dacapo package file if we take some
# precautions (checking md5 sum of the downloaded file)
self.dacapo_md5 = my_config.get('dacapo', 'package_md5')
if not self.dacapo_md5:
e_msg = 'Could not read dacapo package md5sum from conf file'
raise error.TestError(e_msg)
self.dacapo_pkg = \
utils.unmap_url_cache(self.cachedir, self.dacapo_url,
self.dacapo_md5)
# Get jvm package URL
self.jvm_pkg_url = my_config.get(jvm, 'jvm_pkg_url')
if not self.jvm_pkg_url:
raise error.TestError('Could not read java vm URL from conf file')
# Let's cache the jvm package as well
self.jvm_pkg_md5 = my_config.get(jvm, 'package_md5')
if not self.jvm_pkg_md5:
raise error.TestError('Could not read java package_md5 from conf file')
self.jvm_pkg = \
utils.unmap_url_cache(self.cachedir, self.jvm_pkg_url,
self.jvm_pkg_md5)
# Install the jvm pakage
package.install(self.jvm_pkg)
# Basic Java environment variables setup
self.java_root = my_config.get(jvm, 'java_root')
if not self.java_root:
raise error.TestError('Could not read java root dir from conf file')
self.set_java_environment(jvm, self.java_root)
# If use_global is set to 'yes', then we want to use the global
# setting instead of per test settings
if my_config.get('global', 'use_global') == 'yes':
self.iterations = my_config.get('global', 'iterations')
self.workload = my_config.get('global', 'workload')
else:
self.iterations = my_config.get(test, 'iterations')
self.workload = my_config.get(test, 'workload')
self.verbose = '-v '
self.workload = '-s %s ' % self.workload
self.iterations = '-n %s ' % self.iterations
self.scratch = '-scratch %s ' % os.path.join(self.resultsdir, test)
# Compose the arguments string
self.args = self.verbose + self.workload + self.scratch \
+ self.iterations + test
# Execute the actual test
try:
utils.system('java -jar %s %s' % (self.dacapo_pkg, self.args))
except:
e_msg = \
'Test %s has failed, command line options "%s"' % (test, self.args)
raise error.TestError(e_msg)
except EnvironmentError, e:
logging.error(e)
else:
logging.warning("Missing file %s", iso_path)
logging.warning(
"Please download it or put an exsiting copy on the "
"appropriate location")
return
else:
logging.info("Found %s", iso_path)
logging.info("Expected SHA1 sum: %s", hash)
answer = utils.ask(
"Would you like to check %s? It might take a while" % iso_path)
if answer == 'y':
try:
utils.unmap_url_cache(destination, url, hash, method="sha1")
file_ok = True
except EnvironmentError, e:
logging.error(e)
else:
logging.info("File %s present, but chose to not verify it",
iso_path)
return
if file_ok:
logging.info("%s present, with proper checksum", iso_path)
if __name__ == "__main__":
logging_manager.configure_logging(virt_utils.VirtLoggingConfig(),
verbose=True)
def install_lsb_packages(self):
if not self.packages_installed:
# First, we download the LSB DTK manager package, worry about
# installing it later
dtk_manager_arch = self.config.get('dtk-manager',
'arch-%s' % self.arch)
dtk_manager_url = self.config.get('dtk-manager',
'tarball_url') % dtk_manager_arch
if not dtk_manager_url:
raise error.TestError('Could not get DTK manager URL from'
' configuration file')
dtk_md5 = self.config.get('dtk-manager', 'md5-%s' % self.arch)
if dtk_md5:
logging.info('Caching LSB DTK manager RPM')
dtk_manager_pkg = utils.unmap_url_cache(
self.cachedir, dtk_manager_url, dtk_md5)
else:
raise error.TestError('Could not find DTK manager package md5,'
' cannot cache DTK manager tarball')
# Get LSB tarball, cache it and uncompress under autotest srcdir
if self.config.get('lsb', 'override_default_url') == 'no':
lsb_url = self.config.get('lsb', 'tarball_url') % self.arch
else:
lsb_url = self.config.get('lsb', 'tarball_url_alt') % self.arch
if not lsb_url:
raise error.TestError(
'Could not get LSB URL from configuration'
' file')
md5_key = 'md5-%s' % self.arch
lsb_md5 = self.config.get('lsb', md5_key)
if lsb_md5:
logging.info('Caching LSB tarball')
lsb_pkg = utils.unmap_url_cache(self.cachedir, lsb_url,
lsb_md5)
else:
raise error.TestError('Could not find LSB package md5, cannot'
' cache LSB tarball')
utils.extract_tarball_to_dir(lsb_pkg, self.srcdir)
# Lets load a file that contains the list of RPMs
os.chdir(self.srcdir)
if not os.path.isfile('inst-config'):
raise IOError('Could not find file with package info,'
' inst-config')
rpm_file_list = open('inst-config', 'r')
pkg_pattern = re.compile('[A-Za-z0-9_.-]*[.][r][p][m]')
lsb_pkg_list = []
for line in rpm_file_list.readlines():
try:
# We will install lsb-dtk-manager separately, so we can remove
# it from the list of packages
if not 'lsb-dtk-manager' in line:
line = re.findall(pkg_pattern, line)[0]
lsb_pkg_list.append(line)
except Exception:
# If we don't get a match, no problem
pass
# Lets figure out the host distro
distro_pkg_support = package.os_support()
if os.path.isfile('/etc/debian_version') and \
distro_pkg_support['dpkg']:
logging.debug('Debian based distro detected')
if distro_pkg_support['conversion']:
logging.debug('Package conversion supported')
distro_type = 'debian-based'
else:
raise EnvironmentError('Package conversion not supported.'
'Cannot handle LSB package'
' installation')
elif distro_pkg_support['rpm']:
logging.debug('Red Hat based distro detected')
distro_type = 'redhat-based'
else:
logging.error('OS does not seem to be red hat or debian based')
raise EnvironmentError(
'Cannot handle LSB package installation')
# According to the host distro detection, we can install the packages
# using the list previously assembled
if distro_type == 'redhat-based':
logging.info('Installing LSB RPM packages')
package.install(dtk_manager_pkg)
for lsb_rpm in lsb_pkg_list:
package.install(lsb_rpm, nodeps=True)
elif distro_type == 'debian-based':
logging.info('Remember that you must have the following lsb'
' compliance packages installed:')
logging.info(
'lsb-core lsb-cxx lsb-graphics lsb-desktop lsb-qt4'
' lsb-languages lsb-multimedia lsb-printing')
logging.info('Converting and installing LSB packages')
dtk_manager_dpkg = package.convert(dtk_manager_pkg, 'dpkg')
package.install(dtk_manager_dpkg)
for lsb_rpm in lsb_pkg_list:
lsb_dpkg = package.convert(lsb_rpm, 'dpkg')
package.install(lsb_dpkg, nodeps=True)
self.packages_installed = True
file_ok = True
except EnvironmentError, e:
logging.error(e)
else:
logging.warning("Missing file %s", iso_path)
logging.warning("Please download it or put an exsiting copy on the "
"appropriate location")
return
else:
logging.info("Found %s", iso_path)
logging.info("Expected SHA1 sum: %s", hash)
answer = utils.ask("Would you like to check %s? It might take a while" %
iso_path)
if answer == 'y':
try:
utils.unmap_url_cache(destination, url, hash, method="sha1")
file_ok = True
except EnvironmentError, e:
logging.error(e)
else:
logging.info("File %s present, but chose to not verify it",
iso_path)
return
if file_ok:
logging.info("%s present, with proper checksum", iso_path)
if __name__ == "__main__":
logging_manager.configure_logging(virt_utils.VirtLoggingConfig(),
verbose=True)
