def __init__ (self, dest_dir, log, pkg_dirs=None, no_download=False,
copy_files=False) :
self.dest_dir = dest_dir
self.log = log
self.pkg_dirs = pkg_dirs
self.no_download = no_download
self.copy_files = copy_files
self.toolbox = Toolbox()
def fetch_git_repository(package, use_ssh):
""" Download a git repository """
Toolbox.git(package,
git_repositories[package],
destination=os.path.join(os.getcwd(), package),
use_ssh=use_ssh,
verbose=True)
assert op.isdir(package)
def __init__ (self, dest_dir, log, pkg_dirs=None, no_download=False,
copy_files=False) :
self.dest_dir = dest_dir
self.log = log
self.pkg_dirs = pkg_dirs
self.no_download = no_download
self.copy_files = copy_files
self.toolbox = Toolbox()
class fetch_packages(object):
"""
Download manager for the packages defined by this module - this is used by
install_base_packages.py but also for setting up installer bundles.
"""
def __init__(self,
dest_dir,
log,
pkg_dirs=None,
no_download=False,
copy_files=False):
self.dest_dir = dest_dir
self.log = log
self.pkg_dirs = pkg_dirs
self.no_download = no_download
self.copy_files = copy_files
self.toolbox = Toolbox()
def __call__(
self,
pkg_name,
pkg_url=None,
output_file=None,
return_file_and_status=False,
download_url=None, # If given this is the URL used for downloading, otherwise construct using pgk_url and pkg_name
):
if (pkg_url is None):
pkg_url = BASE_CCI_PKG_URL
if (output_file is None):
output_file = pkg_name
os.chdir(self.dest_dir)
print >> self.log, " getting package %s..." % pkg_name
if (self.pkg_dirs is not None) and (len(self.pkg_dirs) > 0):
for pkg_dir in self.pkg_dirs:
static_file = op.join(pkg_dir, pkg_name)
if (op.exists(static_file)):
print >> self.log, " using %s" % static_file
if self.copy_files:
copy_file(static_file,
op.join(self.dest_dir, output_file))
if return_file_and_status:
return op.join(self.dest_dir, output_file), 0
return op.join(self.dest_dir, output_file)
else:
if return_file_and_status:
return static_file, 0
return static_file
if (self.no_download):
if (op.exists(pkg_name)):
print >> self.log, " using ./%s" % pkg_name
if return_file_and_status:
return op.join(self.dest_dir, output_file), 0
return op.join(self.dest_dir, pkg_name)
else:
raise RuntimeError(
("Package '%s' not found on local filesystems. ") %
pkg_name)
full_url = download_url or "%s/%s" % (pkg_url, pkg_name)
if not download_url:
self.log.write(" downloading from %s : " % pkg_url)
size = self.toolbox.download_to_file(full_url,
output_file,
log=self.log)
if (size == -2):
print >> self.log, " using ./%s (cached)" % pkg_name
if return_file_and_status:
return op.join(self.dest_dir, output_file), size
return op.join(self.dest_dir, output_file)
assert (size > 0), pkg_name
if return_file_and_status:
return op.join(self.dest_dir, output_file), size
return op.join(self.dest_dir, output_file)
def fetch_git_repository(package, use_ssh): """ Download a git repository """ Toolbox.git(package, git_repositories[package], destination=os.path.join(os.getcwd(), package), use_ssh=use_ssh, verbose=True) assert op.isdir(package)
class fetch_packages (object) :
"""
Download manager for the packages defined by this module - this is used by
install_base_packages.py but also for setting up installer bundles.
"""
def __init__ (self, dest_dir, log, pkg_dirs=None, no_download=False,
copy_files=False) :
self.dest_dir = dest_dir
self.log = log
self.pkg_dirs = pkg_dirs
self.no_download = no_download
self.copy_files = copy_files
self.toolbox = Toolbox()
def __call__ (self,
pkg_name,
pkg_url=None,
output_file=None,
return_file_and_status=False,
) :
if (pkg_url is None) :
pkg_url = BASE_CCI_PKG_URL
if (output_file is None) :
output_file = pkg_name
os.chdir(self.dest_dir)
print >> self.log, " getting package %s..." % pkg_name
if (self.pkg_dirs is not None) and (len(self.pkg_dirs) > 0) :
for pkg_dir in self.pkg_dirs :
static_file = op.join(pkg_dir, pkg_name)
if (op.exists(static_file)) :
print >> self.log, " using %s" % static_file
if self.copy_files :
copy_file(static_file, op.join(self.dest_dir, output_file))
if return_file_and_status:
return op.join(self.dest_dir, output_file), 0
return op.join(self.dest_dir, output_file)
else :
if return_file_and_status:
return static_file, 0
return static_file
if (self.no_download) :
if (op.exists(pkg_name)) :
print >> self.log, " using ./%s" % pkg_name
if return_file_and_status:
return op.join(self.dest_dir, output_file), 0
return op.join(self.dest_dir, pkg_name)
else :
raise RuntimeError(("Package '%s' not found on local filesystems. ") %
pkg_name)
full_url = "%s/%s" % (pkg_url, pkg_name)
self.log.write(" downloading from %s : " % pkg_url)
size = self.toolbox.download_to_file(full_url, output_file, log=self.log)
if (size == -2):
print >> self.log, " using ./%s (cached)" % pkg_name
if return_file_and_status:
return op.join(self.dest_dir, output_file), size
return op.join(self.dest_dir, output_file)
assert (size > 0), pkg_name
if return_file_and_status:
return op.join(self.dest_dir, output_file), size
return op.join(self.dest_dir, output_file)
class fetch_packages(object):
"""
Download manager for the packages defined by this module - this is used by
install_base_packages.py but also for setting up installer bundles.
"""
def __init__(self,
dest_dir,
log,
pkg_dirs=None,
no_download=False,
copy_files=False):
self.dest_dir = dest_dir
self.log = log
self.pkg_dirs = pkg_dirs
self.no_download = no_download
self.copy_files = copy_files
self.toolbox = Toolbox()
def __call__(
self,
pkg_name,
pkg_url=None,
output_file=None,
return_file_and_status=False,
download_url=None, # If given this is the URL used for downloading, otherwise construct using pkg_url and pkg_name
):
if (pkg_url is None):
pkg_url = BASE_CCI_PKG_URL
if (output_file is None):
output_file = pkg_name
os.chdir(self.dest_dir)
print(" getting package %s..." % pkg_name, file=self.log)
if (self.pkg_dirs is not None) and (len(self.pkg_dirs) > 0):
for pkg_dir in self.pkg_dirs:
static_file = op.join(pkg_dir, pkg_name)
if (op.exists(static_file)):
print(" using %s" % static_file, file=self.log)
if self.copy_files:
copy_file(static_file,
op.join(self.dest_dir, output_file))
if return_file_and_status:
return op.join(self.dest_dir, output_file), 0
return op.join(self.dest_dir, output_file)
else:
if return_file_and_status:
return static_file, 0
return static_file
if (self.no_download):
if (op.exists(pkg_name)):
print(" using ./%s" % pkg_name, file=self.log)
if return_file_and_status:
return op.join(self.dest_dir, output_file), 0
return op.join(self.dest_dir, pkg_name)
else:
raise RuntimeError(
("Package '%s' not found on local filesystems. ") %
pkg_name)
# Generate list of possible URL candidates
if download_url:
if isinstance(download_url, list):
urls = download_url
else:
urls = [download_url]
else:
if isinstance(pkg_url, list):
urls = ["%s/%s" % (p, pkg_name) for p in pkg_url]
else:
urls = ["%s/%s" % (pkg_url, pkg_name)]
for url_attempt in urls:
self.log.write(" downloading from %s : " % url_attempt)
for retry in (3, 3, 0):
try:
size = self.toolbox.download_to_file(url_attempt,
output_file,
log=self.log)
if (size == -2):
print(" using ./%s (cached)" % pkg_name,
file=self.log)
if return_file_and_status:
return op.join(self.dest_dir, output_file), size
return op.join(self.dest_dir, output_file)
assert size > 0, "File %s has size %d" % (pkg_name, size)
if return_file_and_status:
return op.join(self.dest_dir, output_file), size
return op.join(self.dest_dir, output_file)
except Exception as e:
self.log.write(" download failed with %s" % str(e))
if retry:
self.log.write(" retrying in %d seconds" % retry)
time.sleep(retry)
raise RuntimeError("Could not download " + pkg_name)
