def fetch_pkg_file(self, filename, dest_path):
"""
Fetch a package file and save it to the given destination path
git is an SCM, you can download the test directly. No need to fetch
a bz2'd tarball file. However 'filename' is <type>-<name>.tar.bz2
break this up and only fetch <name>.
:type filename: string
:param filename: The filename of the package file to fetch.
:type dest_path: string
:param dest_path: Destination path to download the file to.
"""
logging.info('Fetching %s from %s to %s', filename, self.url,
dest_path)
name, _ = self.pkgmgr.parse_tarball_name(filename)
package_path = self.branch + " " + name
try:
cmd = self.git_archive_cmd_pattern % (self.url, dest_path,
package_path)
result = self.run_command(cmd)
file_exists = self.run_command('ls %s' % dest_path,
_run_command_dargs={
'ignore_status': True
}).exit_status == 0
if not file_exists:
logging.error('git archive failed: %s', result)
raise error.CmdError(cmd, result)
logging.debug('Successfully fetched %s from %s', package_path,
self.url)
except error.CmdError:
raise error.PackageFetchError('%s not found in %s' %
(name, package_path))
def fetch_pkg_file(self, filename, dest_path):
logging.info('Fetching %s from %s to %s', filename, self.url,
dest_path)
# do a quick test to verify the repo is reachable
self._quick_http_test()
# try to retrieve the package via http
package_url = os.path.join(self.url, filename)
try:
cmd = self.wget_cmd_pattern % (package_url, dest_path)
result = self.run_command(cmd)
file_exists = self.run_command(
'ls %s' % dest_path,
_run_command_dargs={'ignore_status': True}).exit_status == 0
if not file_exists:
logging.error('wget failed: %s', result)
raise error.CmdError(cmd, result)
logging.debug('Successfully fetched %s from %s', filename,
package_url)
except error.CmdError:
# remove whatever junk was retrieved when the get failed
self.run_command('rm -f %s' % dest_path)
raise error.PackageFetchError('%s not found in %s' % (filename,
package_url))
def fetch_pkg_file(self, filename, dest_path):
if os.path.exists(dest_path):
os.remove(dest_path)
logging.info('Fetching %s from autoserv to %s', filename, dest_path)
self.job_harness.fetch_package(filename, dest_path)
if os.path.exists(dest_path):
logging.debug('Successfully fetched %s from autoserv', filename)
else:
raise error.PackageFetchError('%s not fetched from autoserv' %
filename)
def fetch_pkg_file(self, filename, dest_path):
logging.info('Fetching %s from %s to %s', filename, self.url,
dest_path)
local_path = os.path.join(self.url, filename)
try:
self.run_command('cp %s %s' % (local_path, dest_path))
logging.debug('Successfully fetched %s from %s', filename,
local_path)
except error.CmdError, e:
raise error.PackageFetchError(
'Package %s could not be fetched from %s' %
(filename, self.url), e)
def _quick_http_test(self):
""" Run a simple 30 second wget on the repository to see if it is
reachable. This avoids the need to wait for a full 10min timeout.
"""
# just make a temp file to write a test fetch into
mktemp = 'mktemp -u /tmp/tmp.XXXXXX'
dest_file_path = self.run_command(mktemp).stdout.strip()
try:
# build up a wget command
http_cmd = self.wget_cmd_pattern % (self.url, dest_file_path)
try:
self.run_command(http_cmd, _run_command_dargs={'timeout': 30})
except Exception, e:
msg = 'HTTP test failed, unable to contact %s: %s'
raise error.PackageFetchError(msg % (self.url, e))
finally:
self.run_command('rm -rf %s' % dest_file_path)
def fetch_pkg(self,
pkg_name,
dest_path,
repo_url=None,
use_checksum=False,
install=False):
'''
Fetch the package into dest_dir from repo_url. By default repo_url
is None and the package is looked in all the repositories specified.
Otherwise it fetches it from the specific repo_url.
pkg_name : name of the package (ex: test-sleeptest.tar.bz2,
dep-gcc.tar.bz2, kernel.1-1.rpm)
repo_url : the URL of the repository where the package is located.
dest_path : complete path of where the package will be fetched to.
use_checksum : This is set to False to fetch the packages.checksum file
so that the checksum comparison is bypassed for the
checksum file itself. This is used internally by the
packaging system. It should be ignored by externals
callers of this method who use it fetch custom packages.
install : install path has unique name and destination requirements
that vary based on the fetcher that is used. So call them
here as opposed to install_pkg.
'''
try:
self._run_command("ls %s" % os.path.dirname(dest_path))
except (error.CmdError, error.AutoservRunError):
raise error.PackageFetchError("Please provide a valid "
"destination: %s " % dest_path)
# See if the package was already fetched earlier, if so
# the checksums need to be compared and the package is now
# fetched only if they differ.
pkg_exists = False
try:
self._run_command("ls %s" % dest_path)
pkg_exists = True
except (error.CmdError, error.AutoservRunError):
pass
# if a repository location is explicitly provided, fetch the package
# from there and return
if repo_url:
repositories = [self.get_fetcher(repo_url)]
elif self.repositories:
repositories = self.repositories
else:
raise error.PackageFetchError("No repository urls specified")
# install the package from the package repos, try the repos in
# reverse order, assuming that the 'newest' repos are most desirable
for fetcher in reversed(repositories):
try:
if isinstance(fetcher, GitFetcher):
use_checksum = False
# different fetchers have different install requirements
dest = fetcher.install_pkg_setup(pkg_name, dest_path,
install)[1]
# Fetch the package if it is not there, the checksum does
# not match, or checksums are disabled entirely
need_to_fetch = (not use_checksum or not pkg_exists or
not self.compare_checksum(dest, fetcher.url))
if need_to_fetch:
fetcher.fetch_pkg_file(pkg_name, dest)
# update checksum so we won't refetch next time.
if use_checksum:
self.update_checksum(dest)
return fetcher
except (error.PackageFetchError, error.AutoservRunError):
# The package could not be found in this repo, continue looking
logging.debug('%s could not be fetched from %s', pkg_name,
fetcher.url)
repo_url_list = [repo.url for repo in repositories]
message = ('%s could not be fetched from any of the repos %s' %
(pkg_name, repo_url_list))
logging.error(message)
# if we got here then that means the package is not found
# in any of the repositories.
raise error.PackageFetchError(message)
