def archive(self, destination):
"""Just moves this archive to the destination."""
if not self.archive_file:
raise NoArchiveFileError("Cannot call archive() before fetching.")
if not extension(destination) == extension(self.archive_file):
raise ValueError("Cannot archive without matching extensions.")
shutil.move(self.archive_file, destination)
def archive(self, destination):
"""Just moves this archive to the destination."""
if not self.archive_file:
raise NoArchiveFileError("Cannot call archive() before fetching.")
if not extension(destination) == extension(self.archive_file):
raise ValueError("Cannot archive without matching extensions.")
shutil.move(self.archive_file, destination)
def split_url_extension(path):
"""Some URLs have a query string, e.g.:
1. https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true
2. http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz
3. https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0
In (1), the query string needs to be stripped to get at the
extension, but in (2) & (3), the filename is IN a single final query
argument.
This strips the URL into three pieces: ``prefix``, ``ext``, and ``suffix``.
The suffix contains anything that was stripped off the URL to
get at the file extension. In (1), it will be ``'?raw=true'``, but
in (2), it will be empty. In (3) the suffix is a parameter that follows
after the file extension, e.g.:
1. ``('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')``
2. ``('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin', '.tar.gz', None)``
3. ``('https://gitlab.kitware.com/vtk/vtk/repository/archive', '.tar.bz2', '?ref=v7.0.0')``
"""
prefix, ext, suffix = path, '', ''
# Strip off sourceforge download suffix.
# e.g. https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download
match = re.search(r'(.*(?:sourceforge\.net|sf\.net)/.*)(/download)$', path)
if match:
prefix, suffix = match.groups()
ext = comp.extension(prefix)
if ext is not None:
prefix = comp.strip_extension(prefix)
else:
prefix, suf = strip_query_and_fragment(prefix)
ext = comp.extension(prefix)
prefix = comp.strip_extension(prefix)
suffix = suf + suffix
if ext is None:
ext = ''
return prefix, ext, suffix
def split_url_extension(path):
"""Some URLs have a query string, e.g.:
1. https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true
2. http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz
3. https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0
In (1), the query string needs to be stripped to get at the
extension, but in (2) & (3), the filename is IN a single final query
argument.
This strips the URL into three pieces: ``prefix``, ``ext``, and ``suffix``.
The suffix contains anything that was stripped off the URL to
get at the file extension. In (1), it will be ``'?raw=true'``, but
in (2), it will be empty. In (3) the suffix is a parameter that follows
after the file extension, e.g.:
1. ``('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')``
2. ``('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin', '.tar.gz', None)``
3. ``('https://gitlab.kitware.com/vtk/vtk/repository/archive', '.tar.bz2', '?ref=v7.0.0')``
"""
prefix, ext, suffix = path, '', ''
# Strip off sourceforge download suffix.
# e.g. https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download
match = re.search(r'(.*(?:sourceforge\.net|sf\.net)/.*)(/download)$', path)
if match:
prefix, suffix = match.groups()
ext = comp.extension(prefix)
if ext is not None:
prefix = comp.strip_extension(prefix)
else:
prefix, suf = strip_query_and_fragment(prefix)
ext = comp.extension(prefix)
prefix = comp.strip_extension(prefix)
suffix = suf + suffix
if ext is None:
ext = ''
return prefix, ext, suffix
def stage(self):
if not self.spec.concrete:
raise ValueError("Can only get a stage for a concrete package.")
if self._stage is None:
# TODO: move this logic into a mirror module.
mirror_path = "%s/%s" % (self.name, "%s-%s.%s" % (
self.name, self.version, extension(self.url)))
self._stage = Stage(
self.url, mirror_path=mirror_path, name=self.spec.short_spec)
return self._stage
def test_native_unpacking(tmpdir_factory, archive_file):
extension = scomp.extension(archive_file)
util = scomp.decompressor_for(archive_file, extension)
tmpdir = tmpdir_factory.mktemp("comp_test")
with working_dir(str(tmpdir)):
assert not os.listdir(os.getcwd())
util(archive_file)
files = os.listdir(os.getcwd())
assert len(files) == 1
with open(files[0], 'r') as f:
contents = f.read()
assert 'TEST' in contents
def split_url_extension(path):
"""Some URLs have a query string, e.g.:
1. https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true
2. http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz
In (1), the query string needs to be stripped to get at the
extension, but in (2), the filename is IN a single final query
argument.
This strips the URL into three pieces: prefix, ext, and suffix.
The suffix contains anything that was stripped off the URL to
get at the file extension. In (1), it will be '?raw=true', but
in (2), it will be empty. e.g.:
1. ('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')
2. ('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin',
'.tar.gz', None)
"""
prefix, ext, suffix = path, '', ''
# Strip off sourceforge download suffix.
match = re.search(r'((?:sourceforge.net|sf.net)/.*)(/download)$', path)
if match:
prefix, suffix = match.groups()
ext = comp.extension(prefix)
if ext is not None:
prefix = comp.strip_extension(prefix)
else:
prefix, suf = strip_query_and_fragment(prefix)
ext = comp.extension(prefix)
prefix = comp.strip_extension(prefix)
suffix = suf + suffix
if ext is None:
ext = ''
return prefix, ext, suffix
def split_url_extension(path):
"""Some URLs have a query string, e.g.:
1. https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true
2. http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz
In (1), the query string needs to be stripped to get at the
extension, but in (2), the filename is IN a single final query
argument.
This strips the URL into three pieces: prefix, ext, and suffix.
The suffix contains anything that was stripped off the URL to
get at the file extension. In (1), it will be '?raw=true', but
in (2), it will be empty. e.g.:
1. ('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')
2. ('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin',
'.tar.gz', None)
"""
prefix, ext, suffix = path, '', ''
# Strip off sourceforge download suffix.
match = re.search(r'((?:sourceforge.net|sf.net)/.*)(/download)$', path)
if match:
prefix, suffix = match.groups()
ext = comp.extension(prefix)
if ext is not None:
prefix = comp.strip_extension(prefix)
else:
prefix, suf = strip_query_and_fragment(prefix)
ext = comp.extension(prefix)
prefix = comp.strip_extension(prefix)
suffix = suf + suffix
if ext is None:
ext = ''
return prefix, ext, suffix
def mirror_archive_filename(spec):
"""Get the path that this spec will live at within a mirror."""
if not spec.version.concrete:
raise ValueError("mirror.path requires spec with concrete version.")
fetcher = spec.package.fetcher
if isinstance(fetcher, fs.URLFetchStrategy):
# If we fetch this version with a URLFetchStrategy, use URL's archive type
ext = extension(fetcher.url)
else:
# Otherwise we'll make a .tar.gz ourselves
ext = 'tar.gz'
return "%s-%s.%s" % (spec.package.name, spec.version, ext)
def archive(self, destination, **kwargs):
assert (extension(destination) == 'tar.gz')
assert (self.stage.source_path.startswith(self.stage.path))
tar = which('tar', required=True)
patterns = kwargs.get('exclude', None)
if patterns is not None:
if isinstance(patterns, basestring):
patterns = [patterns]
for p in patterns:
tar.add_default_arg('--exclude=%s' % p)
self.stage.chdir()
tar('-czf', destination, os.path.basename(self.stage.source_path))
def archive(self, destination, **kwargs):
assert (extension(destination) == 'tar.gz')
assert (self.stage.source_path.startswith(self.stage.path))
tar = which('tar', required=True)
patterns = kwargs.get('exclude', None)
if patterns is not None:
if isinstance(patterns, basestring):
patterns = [patterns]
for p in patterns:
tar.add_default_arg('--exclude=%s' % p)
self.stage.chdir()
tar('-czf', destination, os.path.basename(self.stage.source_path))
def expand(self):
if not self.expand_archive:
tty.msg("Skipping expand step for %s" % self.archive_file)
return
tty.msg("Staging archive: %s" % self.archive_file)
if not self.archive_file:
raise NoArchiveFileError(
"Couldn't find archive file",
"Failed on expand() for URL %s" % self.url)
if not self.extension:
self.extension = extension(self.archive_file)
decompress = decompressor_for(self.archive_file, self.extension)
# Expand all tarballs in their own directory to contain
# exploding tarballs.
tarball_container = os.path.join(self.stage.path,
"spack-expanded-archive")
mkdirp(tarball_container)
with working_dir(tarball_container):
decompress(self.archive_file)
# Check for an exploding tarball, i.e. one that doesn't expand
# to a single directory. If the tarball *didn't* explode,
# move contents up & remove the container directory.
#
# NOTE: The tar program on Mac OS X will encode HFS metadata
# in hidden files, which can end up *alongside* a single
# top-level directory. We ignore hidden files to accomodate
# these "semi-exploding" tarballs.
files = os.listdir(tarball_container)
non_hidden = [f for f in files if not f.startswith('.')]
if len(non_hidden) == 1:
expanded_dir = os.path.join(tarball_container, non_hidden[0])
if os.path.isdir(expanded_dir):
for f in files:
shutil.move(os.path.join(tarball_container, f),
os.path.join(self.stage.path, f))
os.rmdir(tarball_container)
if not files:
os.rmdir(tarball_container)
def expand(self):
if not self.expand_archive:
tty.msg("Skipping expand step for %s" % self.archive_file)
return
tty.msg("Staging archive: %s" % self.archive_file)
self.stage.chdir()
if not self.archive_file:
raise NoArchiveFileError(
"Couldn't find archive file",
"Failed on expand() for URL %s" % self.url)
if not self.extension:
self.extension = extension(self.archive_file)
decompress = decompressor_for(self.archive_file, self.extension)
# Expand all tarballs in their own directory to contain
# exploding tarballs.
tarball_container = os.path.join(self.stage.path,
"spack-expanded-archive")
mkdirp(tarball_container)
os.chdir(tarball_container)
decompress(self.archive_file)
# Check for an exploding tarball, i.e. one that doesn't expand
# to a single directory. If the tarball *didn't* explode,
# move contents up & remove the container directory.
#
# NOTE: The tar program on Mac OS X will encode HFS metadata
# in hidden files, which can end up *alongside* a single
# top-level directory. We ignore hidden files to accomodate
# these "semi-exploding" tarballs.
files = os.listdir(tarball_container)
non_hidden = [f for f in files if not f.startswith('.')]
if len(non_hidden) == 1:
expanded_dir = os.path.join(tarball_container, non_hidden[0])
if os.path.isdir(expanded_dir):
for f in files:
shutil.move(os.path.join(tarball_container, f),
os.path.join(self.stage.path, f))
os.rmdir(tarball_container)
if not files:
os.rmdir(tarball_container)
# Set the wd back to the stage when done.
self.stage.chdir()
def stage(self):
if not self.spec.concrete:
raise ValueError("Can only get a stage for a concrete package.")
if self._stage is None:
if not self.url:
raise PackageVersionError(self.version)
# TODO: move this logic into a mirror module.
mirror_path = "%s/%s" % (
self.name, "%s-%s.%s" %
(self.name, self.version, extension(self.url)))
self._stage = Stage(self.url,
mirror_path=mirror_path,
name=self.spec.short_spec)
return self._stage
def archive(self, destination, **kwargs):
assert (extension(destination) == 'tar.gz')
assert (self.stage.source_path.startswith(self.stage.path))
tar = which('tar', required=True)
patterns = kwargs.get('exclude', None)
if patterns is not None:
if isinstance(patterns, string_types):
patterns = [patterns]
for p in patterns:
tar.add_default_arg('--exclude=%s' % p)
with working_dir(self.stage.path):
if self.stage.srcdir:
# Here we create an archive with the default repository name.
# The 'tar' command has options for changing the name of a
# directory that is included in the archive, but they differ
# based on OS, so we temporarily rename the repo
with temp_rename(self.stage.source_path, self.stage.srcdir):
tar('-czf', destination, self.stage.srcdir)
else:
tar('-czf', destination,
os.path.basename(self.stage.source_path))
def substitution_offsets(path):
"""This returns offsets for substituting versions and names in the provided path.
It is a helper for substitute_version().
"""
# Get name and version offsets
try:
ver, vs, vl = parse_version_offset(path)
name, ns, nl = parse_name_offset(path, ver)
except UndetectableNameError, e:
return (None, -1, -1, (), ver, vs, vl, (vs, ))
except UndetectableVersionError, e:
return (None, -1, -1, (), None, -1, -1, ())
# protect extensions like bz2 from getting inadvertently
# considered versions.
ext = comp.extension(path)
path = comp.strip_extension(path)
# Construct a case-insensitive regular expression for the package name.
name_re = '(%s)' % insensitize(name)
# Split the string apart by things that match the name so that if the
# name contains numbers or things that look like versions, we don't
# accidentally substitute them with a version.
name_parts = re.split(name_re, path)
offsets = cumsum(name_parts, 0, len)
name_offsets = offsets[1::2]
ver_offsets = []
for i in xrange(0, len(name_parts), 2):
def test_get_extension(archive):
ext = scomp.extension(archive)
assert ext_archive[ext] == archive
def test_get_bad_extension():
archive = 'Foo.py'
ext = scomp.extension(archive)
assert ext is None
% pkg.name)
continue
# create a subdir for the current package.
pkg_path = join_path(args.directory, pkg.name)
mkdirp(pkg_path)
# Download all the tarballs using Stages, then move them into place
for version in pkg.versions:
# Skip versions that don't match the spec
vspec = Spec('%s@%s' % (pkg.name, version))
if not vspec.satisfies(spec):
continue
mirror_path = "%s/%s-%s.%s" % (
pkg.name, pkg.name, version, extension(pkg.url))
os.chdir(working_dir)
mirror_file = join_path(args.directory, mirror_path)
if os.path.exists(mirror_file):
tty.msg("Already fetched %s." % mirror_file)
num_mirrored += 1
continue
# Get the URL for the version and set up a stage to download it.
url = pkg.url_for_version(version)
stage = Stage(url)
try:
# fetch changes directory into the stage
stage.fetch()
def substitution_offsets(path):
"""This returns offsets for substituting versions and names in the provided path.
It is a helper for substitute_version().
"""
# Get name and version offsets
try:
ver, vs, vl = parse_version_offset(path)
name, ns, nl = parse_name_offset(path, ver)
except UndetectableNameError, e:
return (None, -1, -1, (), ver, vs, vl, (vs,))
except UndetectableVersionError, e:
return (None, -1, -1, (), None, -1, -1, ())
# protect extensions like bz2 from getting inadvertently
# considered versions.
ext = comp.extension(path)
path = comp.strip_extension(path)
# Construct a case-insensitive regular expression for the package name.
name_re = '(%s)' % insensitize(name)
# Split the string apart by things that match the name so that if the
# name contains numbers or things that look like versions, we don't
# accidentally substitute them with a version.
name_parts = re.split(name_re, path)
offsets = cumsum(name_parts, 0, len)
name_offsets = offsets[1::2]
ver_offsets = []
for i in xrange(0, len(name_parts), 2):
pkg.name)
continue
# create a subdir for the current package.
pkg_path = join_path(args.directory, pkg.name)
mkdirp(pkg_path)
# Download all the tarballs using Stages, then move them into place
for version in pkg.versions:
# Skip versions that don't match the spec
vspec = Spec('%s@%s' % (pkg.name, version))
if not vspec.satisfies(spec):
continue
mirror_path = "%s/%s-%s.%s" % (pkg.name, pkg.name, version,
extension(pkg.url))
os.chdir(working_dir)
mirror_file = join_path(args.directory, mirror_path)
if os.path.exists(mirror_file):
tty.msg("Already fetched %s." % mirror_file)
num_mirrored += 1
continue
# Get the URL for the version and set up a stage to download it.
url = pkg.url_for_version(version)
stage = Stage(url)
try:
# fetch changes directory into the stage
stage.fetch()
def expand(self):
if not self.expand_archive:
tty.msg("Staging unexpanded archive %s in %s" %
(self.archive_file, self.stage.source_path))
if not self.stage.expanded:
mkdirp(self.stage.source_path)
dest = os.path.join(self.stage.source_path,
os.path.basename(self.archive_file))
# if the archive is a symlink itself, copy the target because
# otherwise the symlink target might get modified by
# staging-operations
if os.path.islink(self.archive_file):
shutil.copy(self.archive_file, dest)
else:
shutil.move(self.archive_file, dest)
return
tty.msg("Staging archive: %s" % self.archive_file)
if not self.archive_file:
raise NoArchiveFileError(
"Couldn't find archive file",
"Failed on expand() for URL %s" % self.url)
if not self.extension:
self.extension = extension(self.archive_file)
if self.stage.expanded:
tty.debug('Source already staged to %s' % self.stage.source_path)
return
decompress = decompressor_for(self.archive_file, self.extension)
# Expand all tarballs in their own directory to contain
# exploding tarballs.
tarball_container = os.path.join(self.stage.path,
"spack-expanded-archive")
mkdirp(tarball_container)
with working_dir(tarball_container):
decompress(self.archive_file)
# Check for an exploding tarball, i.e. one that doesn't expand to
# a single directory. If the tarball *didn't* explode, move its
# contents to the staging source directory & remove the container
# directory. If the tarball did explode, just rename the tarball
# directory to the staging source directory.
#
# NOTE: The tar program on Mac OS X will encode HFS metadata in
# hidden files, which can end up *alongside* a single top-level
# directory. We initially ignore presence of hidden files to
# accomodate these "semi-exploding" tarballs but ensure the files
# are copied to the source directory.
files = os.listdir(tarball_container)
non_hidden = [f for f in files if not f.startswith('.')]
if len(non_hidden) == 1:
src = os.path.join(tarball_container, non_hidden[0])
if os.path.isdir(src):
self.stage.srcdir = non_hidden[0]
shutil.move(src, self.stage.source_path)
if len(files) > 1:
files.remove(non_hidden[0])
for f in files:
src = os.path.join(tarball_container, f)
dest = os.path.join(self.stage.path, f)
shutil.move(src, dest)
os.rmdir(tarball_container)
else:
# This is a non-directory entry (e.g., a patch file) so simply
# rename the tarball container to be the source path.
shutil.move(tarball_container, self.stage.source_path)
else:
shutil.move(tarball_container, self.stage.source_path)
