def download(self): """ Download kernel source. """ self.kernel_file = KernelBuild.source.format(version=self.version) full_url = KernelBuild.url + KernelBuild.source.format(version=self.version) path = os.path.join(self.work_dir, self.kernel_file) if os.path.isfile(path): log.info("File '%s' exists, will not download!", path) else: log.info("Downloading '%s'...", full_url) download.url_download(full_url, path)
def download(self): """ Download kernel source. """ self.kernel_file = KernelBuild.source.format(version=self.version) full_url = KernelBuild.url + KernelBuild.source.format( version=self.version) path = os.path.join(self.work_dir, self.kernel_file) if os.path.isfile(path): log.info("File '%s' exists, will not download!", path) else: log.info("Downloading '%s'...", full_url) download.url_download(full_url, path)
def __init__(self, cfg, tmpdir=data_dir.get_tmp_dir(), raise_errors=False): """ Instantiate configparser and load data. :param cfg: Where we'll get configuration data. It can be either: * A URL containing the file * A valid file path inside the filesystem * A string containing configuration data :param tmpdir: Where we'll dump the temporary conf files. :param raise_errors: Whether config value absences will raise ValueError exceptions. """ # Base Parser self.parser = ConfigParser.ConfigParser() # Raise errors when lacking values self.raise_errors = raise_errors # File is already a file like object if hasattr(cfg, 'read'): self.cfg = cfg self.parser.readfp(self.cfg) elif isinstance(cfg, string_types): # Config file is a URL. Download it to a temp dir if cfg.startswith('http') or cfg.startswith('ftp'): self.cfg = os.path.join(tmpdir, os.path.basename(cfg)) download.url_download(cfg, self.cfg) self.parser.read(self.cfg) # Config is a valid filesystem path to a file. elif os.path.exists(os.path.abspath(cfg)): if os.path.isfile(cfg): self.cfg = os.path.abspath(cfg) self.parser.read(self.cfg) else: e_msg = 'Invalid config file path: %s' % cfg raise IOError(e_msg) # Config file is just a string, convert it to a python file like # object using StringIO else: self.cfg = StringIO(cfg) self.parser.readfp(self.cfg)
def __init__(self, cfg, tmpdir=data_dir.get_tmp_dir(), raise_errors=False): """ Instantiate ConfigParser and load data. :param cfg: Where we'll get configuration data. It can be either: * A URL containing the file * A valid file path inside the filesystem * A string containing configuration data :param tmpdir: Where we'll dump the temporary conf files. :param raise_errors: Whether config value absences will raise ValueError exceptions. """ # Base Parser self.parser = ConfigParser.ConfigParser() # Raise errors when lacking values self.raise_errors = raise_errors # File is already a file like object if hasattr(cfg, 'read'): self.cfg = cfg self.parser.readfp(self.cfg) elif isinstance(cfg, types.StringTypes): # Config file is a URL. Download it to a temp dir if cfg.startswith('http') or cfg.startswith('ftp'): self.cfg = os.path.join(tmpdir, os.path.basename(cfg)) download.url_download(cfg, self.cfg) self.parser.read(self.cfg) # Config is a valid filesystem path to a file. elif os.path.exists(os.path.abspath(cfg)): if os.path.isfile(cfg): self.cfg = os.path.abspath(cfg) self.parser.read(self.cfg) else: e_msg = 'Invalid config file path: %s' % cfg raise IOError(e_msg) # Config file is just a string, convert it to a python file like # object using StringIO else: self.cfg = StringIO.StringIO(cfg) self.parser.readfp(self.cfg)
def _download(self, url_obj, asset_path, timeout=None): """ Download the asset from an uri. :param url_obj: object from urlparse. :param asset_path: full path of the asset file. :param timeout: timeout in seconds. Default is :data:`avocado.utils.asset.DOWNLOAD_TIMEOUT`. :returns: if the downloaded file matches the hash. :rtype: bool """ timeout = timeout or DOWNLOAD_TIMEOUT try: # Temporary unique name to use while downloading temp = '%s.%s' % (asset_path, str(uuid.uuid4())) # To avoid parallel downloads of the same asset, and errors during # the write after download, let's get the lock before start the # download. with FileLock(asset_path, 120): try: self.find_asset_file(create_metadata=True) return True except OSError: LOG.debug("Asset not in cache after lock, fetching it.") url_download(url_obj.geturl(), temp, timeout=timeout) shutil.copy(temp, asset_path) self._create_hash_file(asset_path) if not self._verify_hash(asset_path): msg = "Hash mismatch. Ignoring asset from the cache" raise OSError(msg) return True finally: try: os.remove(temp) except FileNotFoundError: LOG.info("Temporary asset file unavailable due to failed" " download attempt.")
class VirtBootstrap(plugin.Plugin): """ Implements the avocado 'virt-bootstrap' subcommand """ name = 'virt_bootstrap' enabled = True def configure(self, parser): self.parser = parser.subcommands.add_parser( 'virt-bootstrap', help='Download image files important to avocado virt tests') super(VirtBootstrap, self).configure(self.parser) def run(self, args): fail = False view = output.View(app_args=args) view.notify(event='message', msg='Probing your system for test requirements') try: utils_path.find_command('7za') view.notify(event='minor', msg='7zip present') except utils_path.CmdNotFoundError: view.notify(event='warning', msg=("7za not installed. You may " "install 'p7zip' (or the " "equivalent on your distro) to " "fix the problem")) fail = True jeos_sha1_url = 'https://lmr.fedorapeople.org/jeos/SHA1SUM_JEOS20' try: view.notify(event='minor', msg=('Verifying expected SHA1 ' 'sum from %s' % jeos_sha1_url)) sha1_file = urllib2.urlopen(jeos_sha1_url) sha1_contents = sha1_file.read() sha1 = sha1_contents.split(" ")[0] view.notify(event='minor', msg='Expected SHA1 sum: %s' % sha1) except Exception, e: view.notify(event='error', msg='Failed to get SHA1 from file: %s' % e) fail = True jeos_dst_dir = path.init_dir( os.path.join(data_dir.get_data_dir(), 'images')) jeos_dst_path = os.path.join(jeos_dst_dir, 'jeos-20-64.qcow2.7z') if os.path.isfile(jeos_dst_path): actual_sha1 = crypto.hash_file(filename=jeos_dst_path, algorithm="sha1") else: actual_sha1 = '0' if actual_sha1 != sha1: if actual_sha1 == '0': view.notify(event='minor', msg=('JeOS could not be found at %s. Downloading ' 'it (173 MB). Please wait...' % jeos_dst_path)) else: view.notify(event='minor', msg=('JeOS at %s is either corrupted or outdated. ' 'Downloading a new copy (173 MB). ' 'Please wait...' % jeos_dst_path)) jeos_url = 'https://lmr.fedorapeople.org/jeos/jeos-20-64.qcow2.7z' try: download.url_download(jeos_url, jeos_dst_path) except: view.notify(event='warning', msg=('Exiting upon user request (Download ' 'not finished)')) else: view.notify(event='minor', msg=('Compressed JeOS image found ' 'in %s, with proper SHA1' % jeos_dst_path)) view.notify(event='minor', msg=('Uncompressing the JeOS image to restore pristine ' 'state. Please wait...')) os.chdir(os.path.dirname(jeos_dst_path)) result = process.run('7za -y e %s' % os.path.basename(jeos_dst_path), ignore_status=True) if result.exit_status != 0: view.notify(event='error', msg=('Error uncompressing the image ' '(see details below):\n%s' % result)) fail = True else: view.notify(event='minor', msg='Successfully uncompressed the image') if fail: view.notify(event='warning', msg=('Problems found probing this system for tests ' 'requirements. Please check the error messages ' 'and fix the problems found')) else: view.notify(event='message', msg=('Your system appears to be all ' 'set to execute tests'))
class VirtBootstrap(CLICmd): """ Implements the avocado 'virt-bootstrap' subcommand """ name = 'virt-bootstrap' description = "Avocado-Virt 'virt-bootstrap' subcommand" def run(self, args): fail = False LOG.info('Probing your system for test requirements') try: utils_path.find_command('xz') logging.debug('xz present') except utils_path.CmdNotFoundError: LOG.warn("xz not installed. You may install xz (or the " "equivalent on your distro) to fix the problem") fail = True jeos_sha1_url = ("https://avocado-project.org/data/assets/jeos/25/" "SHA1SUM_JEOS25") try: LOG.debug('Verifying expected SHA1 sum from %s', jeos_sha1_url) sha1_file = urllib2.urlopen(jeos_sha1_url) sha1_contents = sha1_file.read() sha1 = sha1_contents.split(" ")[0] LOG.debug('Expected SHA1 sum: %s', sha1) except Exception, exc: LOG.error('Failed to get SHA1 from file: %s', exc) fail = True sha1 = "FAILED TO GET DOWNLOADED FROM AVOCADO-PROJECT" jeos_dst_dir = path.init_dir( os.path.join(data_dir.get_data_dir(), 'images')) jeos_dst_path = os.path.join(jeos_dst_dir, 'jeos-25-64.qcow2.xz') if os.path.isfile(jeos_dst_path): actual_sha1 = crypto.hash_file(filename=jeos_dst_path, algorithm="sha1") else: actual_sha1 = 'FILE DOES NOT EXIST LOCALLY' if actual_sha1 != sha1: if actual_sha1 == 'FILE DOES NOT EXIST LOCALLY': LOG.debug( 'JeOS could not be found at %s. Downloading ' 'it (205 MB). Please wait...', jeos_dst_path) else: LOG.debug( 'JeOS at %s is either corrupted or outdated. ' 'Downloading a new copy (205 MB). ' 'Please wait...', jeos_dst_path) jeos_url = ("https://avocado-project.org/data/assets/jeos/25/" "jeos-25-64.qcow2.xz") try: download.url_download(jeos_url, jeos_dst_path) except: LOG.warn('Exiting upon user request (Download not finished)') else: LOG.debug('Compressed JeOS image found in %s, with proper SHA1', jeos_dst_path) LOG.debug('Uncompressing the JeOS image to restore pristine ' 'state. Please wait...') os.chdir(os.path.dirname(jeos_dst_path)) cmd = 'xz --keep --force -d %s' % os.path.basename(jeos_dst_path) result = process.run(cmd, ignore_status=True) if result.exit_status != 0: LOG.error('Error uncompressing the image (see details below):\n%s', result) fail = True else: LOG.debug('Successfully uncompressed the image') if fail: LOG.warn('Problems found probing this system for tests ' 'requirements. Please check the error messages ' 'and fix the problems found') else: LOG.info('Your system appears to be all set to execute tests')