def fetch_pkg_file(self, filename, dest_path): """ Fetch a package file and save it to the given destination path git is an SCM, you can download the test directly. No need to fetch a bz2'd tarball file. However 'filename' is <type>-<name>.tar.bz2 break this up and only fetch <name>. :type filename: string :param filename: The filename of the package file to fetch. :type dest_path: string :param dest_path: Destination path to download the file to. """ logging.info('Fetching %s from %s to %s', filename, self.url, dest_path) name, _ = self.pkgmgr.parse_tarball_name(filename) package_path = self.branch + " " + name try: cmd = self.git_archive_cmd_pattern % (self.url, dest_path, package_path) result = self.run_command(cmd) file_exists = self.run_command('ls %s' % dest_path, _run_command_dargs={ 'ignore_status': True }).exit_status == 0 if not file_exists: logging.error('git archive failed: %s', result) raise error.CmdError(cmd, result) logging.debug('Successfully fetched %s from %s', package_path, self.url) except error.CmdError: raise error.PackageFetchError('%s not found in %s' % (name, package_path))
def fetch_pkg_file(self, filename, dest_path): logging.info('Fetching %s from %s to %s', filename, self.url, dest_path) # do a quick test to verify the repo is reachable self._quick_http_test() # try to retrieve the package via http package_url = os.path.join(self.url, filename) try: cmd = self.wget_cmd_pattern % (package_url, dest_path) result = self.run_command(cmd) file_exists = self.run_command( 'ls %s' % dest_path, _run_command_dargs={'ignore_status': True}).exit_status == 0 if not file_exists: logging.error('wget failed: %s', result) raise error.CmdError(cmd, result) logging.debug('Successfully fetched %s from %s', filename, package_url) except error.CmdError: # remove whatever junk was retrieved when the get failed self.run_command('rm -f %s' % dest_path) raise error.PackageFetchError('%s not found in %s' % (filename, package_url))
def fetch_pkg_file(self, filename, dest_path): if os.path.exists(dest_path): os.remove(dest_path) logging.info('Fetching %s from autoserv to %s', filename, dest_path) self.job_harness.fetch_package(filename, dest_path) if os.path.exists(dest_path): logging.debug('Successfully fetched %s from autoserv', filename) else: raise error.PackageFetchError('%s not fetched from autoserv' % filename)
def fetch_pkg_file(self, filename, dest_path): logging.info('Fetching %s from %s to %s', filename, self.url, dest_path) local_path = os.path.join(self.url, filename) try: self.run_command('cp %s %s' % (local_path, dest_path)) logging.debug('Successfully fetched %s from %s', filename, local_path) except error.CmdError, e: raise error.PackageFetchError( 'Package %s could not be fetched from %s' % (filename, self.url), e)
def _quick_http_test(self): """ Run a simple 30 second wget on the repository to see if it is reachable. This avoids the need to wait for a full 10min timeout. """ # just make a temp file to write a test fetch into mktemp = 'mktemp -u /tmp/tmp.XXXXXX' dest_file_path = self.run_command(mktemp).stdout.strip() try: # build up a wget command http_cmd = self.wget_cmd_pattern % (self.url, dest_file_path) try: self.run_command(http_cmd, _run_command_dargs={'timeout': 30}) except Exception, e: msg = 'HTTP test failed, unable to contact %s: %s' raise error.PackageFetchError(msg % (self.url, e)) finally: self.run_command('rm -rf %s' % dest_file_path)
def fetch_pkg(self, pkg_name, dest_path, repo_url=None, use_checksum=False, install=False): ''' Fetch the package into dest_dir from repo_url. By default repo_url is None and the package is looked in all the repositories specified. Otherwise it fetches it from the specific repo_url. pkg_name : name of the package (ex: test-sleeptest.tar.bz2, dep-gcc.tar.bz2, kernel.1-1.rpm) repo_url : the URL of the repository where the package is located. dest_path : complete path of where the package will be fetched to. use_checksum : This is set to False to fetch the packages.checksum file so that the checksum comparison is bypassed for the checksum file itself. This is used internally by the packaging system. It should be ignored by externals callers of this method who use it fetch custom packages. install : install path has unique name and destination requirements that vary based on the fetcher that is used. So call them here as opposed to install_pkg. ''' try: self._run_command("ls %s" % os.path.dirname(dest_path)) except (error.CmdError, error.AutoservRunError): raise error.PackageFetchError("Please provide a valid " "destination: %s " % dest_path) # See if the package was already fetched earlier, if so # the checksums need to be compared and the package is now # fetched only if they differ. pkg_exists = False try: self._run_command("ls %s" % dest_path) pkg_exists = True except (error.CmdError, error.AutoservRunError): pass # if a repository location is explicitly provided, fetch the package # from there and return if repo_url: repositories = [self.get_fetcher(repo_url)] elif self.repositories: repositories = self.repositories else: raise error.PackageFetchError("No repository urls specified") # install the package from the package repos, try the repos in # reverse order, assuming that the 'newest' repos are most desirable for fetcher in reversed(repositories): try: if isinstance(fetcher, GitFetcher): use_checksum = False # different fetchers have different install requirements dest = fetcher.install_pkg_setup(pkg_name, dest_path, install)[1] # Fetch the package if it is not there, the checksum does # not match, or checksums are disabled entirely need_to_fetch = (not use_checksum or not pkg_exists or not self.compare_checksum(dest, fetcher.url)) if need_to_fetch: fetcher.fetch_pkg_file(pkg_name, dest) # update checksum so we won't refetch next time. if use_checksum: self.update_checksum(dest) return fetcher except (error.PackageFetchError, error.AutoservRunError): # The package could not be found in this repo, continue looking logging.debug('%s could not be fetched from %s', pkg_name, fetcher.url) repo_url_list = [repo.url for repo in repositories] message = ('%s could not be fetched from any of the repos %s' % (pkg_name, repo_url_list)) logging.error(message) # if we got here then that means the package is not found # in any of the repositories. raise error.PackageFetchError(message)