"""

Bootstrapper implementation. See ``use_astropy_helpers`` for parameter

documentation.

"""

def __init__(self, path=None, index_url=None, use_git=None, offline=None,

download_if_needed=None, auto_upgrade=None):

if path is None:

path = PACKAGE_NAME

if not (isinstance(path, _str_types) or path is False):

raise TypeError('path must be a string or False')

if PY3 and not isinstance(path, _text_type):

fs_encoding = sys.getfilesystemencoding()

path = path.decode(fs_encoding) # path to unicode

self.path = path

# Set other option attirbutes, using defaults where necessary

self.index_url = index_url if index_url is not None else INDEX_URL

self.offline = offline if offline is not None else OFFLINE

# If offline=True, override download and auto-upgrade

if self.offline:

download_if_needed = False

auto_upgrade = False

self.download = (download_if_needed

if download_if_needed is not None

else DOWNLOAD_IF_NEEDED)

self.auto_upgrade = (auto_upgrade

if auto_upgrade is not None else AUTO_UPGRADE)

# If this is a release then the .git directory will not exist so we

# should not use git.

git_dir_exists = os.path.exists(os.path.join(os.path.dirname(__file__), '.git'))

if use_git is None and not git_dir_exists:

use_git = False

self.use_git = use_git if use_git is not None else USE_GIT

# Declared as False by default--later we check if astropy-helpers can be

# upgraded from PyPI, but only if not using a source distribution (as in

# the case of import from a git submodule)

self.is_submodule = False

@classmethod

def main(cls, args=None):

if args is None:

args = sys.argv[1:]

config = cls.parse_config()

config.update(cls.parse_command_line(args))

auto_use = config.pop('auto_use', False)

if auto_use:

bootstrapper = cls(**config)

bootstrapper.run()

return config

@classmethod

def parse_config(cls):

cfg_options = [('auto_use', bool), ('path', str),

('download_if_needed', bool), ('index_url', str),

('use_git', bool), ('auto_upgrade', bool)]

if not os.path.exists('setup.cfg'):

return {}

cfg = ConfigParser()

try:

cfg.read('setup.cfg')

except Exception as e:

if DEBUG:

raise

log.error(

"Error reading setup.cfg: {0!r}\n{1} will not be "

"automatically bootstrapped and package installation may fail."

"\n{2}".format(e, PACKAGE_NAME, _err_help_msg))

return {}

if not cfg.has_section('ah_bootstrap'):

return {}

config = {}

for option, type_ in cfg_options:

if not cfg.has_option('ah_bootstrap', option):

continue

if type_ is bool:

value = cfg.getboolean('ah_bootstrap', option)

else:

value = cfg.get('ah_bootstrap', option)

config[option] = value

return config

@classmethod

def parse_command_line(cls, args=None):

if args is None:

args = sys.argv[1:]

config = {}

# For now we just pop recognized ah_bootstrap options out of the

# arg list. This is imperfect; in the unlikely case that a setup.py

# custom command or even custom Distribution class defines an argument

# of the same name then we will break that. However there's a catch22

# here that we can't just do full argument parsing right here, because

# we don't yet know *how* to parse all possible command-line arguments.

if '--no-git' in args:

config['use_git'] = False

args.remove('--no-git')

if '--offline' in args:

config['offline'] = True

args.remove('--offline')

return config

def run(self):

strategies = ['local_directory', 'local_file', 'index']

dist = None

# Check to see if the path is a submodule

self.is_submodule = self._check_submodule()

for strategy in strategies:

method = getattr(self, 'get_{0}_dist'.format(strategy))

dist = method()

if dist is not None:

break

else:

raise _AHBootstrapSystemExit(

"No source found for the {0!r} package; {0} must be "

"available and importable as a prerequisite to building "

"or installing this package.".format(PACKAGE_NAME))

# Otherwise we found a version of astropy-helpers, so we're done

# Just active the found distribution on sys.path--if we did a

# download this usually happens automatically but it doesn't hurt to

# do it again

# Note: Adding the dist to the global working set also activates it

# (makes it importable on sys.path) by default

pkg_resources.working_set.add(dist)

def get_local_directory_dist(self):

"""

Handle importing a vendored package from a subdirectory of the source

distribution.

"""

if not os.path.isdir(self.path):

return

log.info('Attempting to import astropy_helpers from {0} {1!r}'.format(

'submodule' if self.is_submodule else 'directory',

self.path))

dist = self._directory_import()

if dist is None:

log.warn(

'The requested path {0!r} for importing {1} does not '

'exist, or does not contain a copy of the {1} '

'package.'.format(self.path, PACKAGE_NAME))

elif self.auto_upgrade and not self.is_submodule:

# A version of astropy-helpers was found on the available path, but

# check to see if a bugfix release is available on PyPI

upgrade = self._do_upgrade(dist)

if upgrade is not None:

dist = upgrade

return dist

def get_local_file_dist(self):

"""

Handle importing from a source archive; this also uses setup_requires

but points easy_install directly to the source archive.

"""

if not os.path.isfile(self.path):

return

log.info('Attempting to unpack and import astropy_helpers from '

'{0!r}'.format(self.path))

try:

dist = self._do_download(find_links=[self.path])

except Exception as e:

if DEBUG:

raise

log.warn(

'Failed to import {0} from the specified archive {1!r}: '

'{2}'.format(PACKAGE_NAME, self.path, str(e)))

dist = None

if dist is not None and self.auto_upgrade:

# A version of astropy-helpers was found on the available path, but

# check to see if a bugfix release is available on PyPI

upgrade = self._do_upgrade(dist)

if upgrade is not None:

dist = upgrade

return dist

def get_index_dist(self):

if not self.download:

log.warn('Downloading {0!r} disabled.'.format(DIST_NAME))

return False

log.warn(

"Downloading {0!r}; run setup.py with the --offline option to "

"force offline installation.".format(DIST_NAME))

try:

dist = self._do_download()

except Exception as e:

if DEBUG:

raise

log.warn(

'Failed to download and/or install {0!r} from {1!r}:\n'

'{2}'.format(DIST_NAME, self.index_url, str(e)))

dist = None

# No need to run auto-upgrade here since we've already presumably

# gotten the most up-to-date version from the package index

return dist

def _directory_import(self):

"""

Import astropy_helpers from the given path, which will be added to

sys.path.

Must return True if the import succeeded, and False otherwise.

"""

# Return True on success, False on failure but download is allowed, and

# otherwise raise SystemExit

path = os.path.abspath(self.path)

# Use an empty WorkingSet rather than the man

# pkg_resources.working_set, since on older versions of setuptools this

# will invoke a VersionConflict when trying to install an upgrade

ws = pkg_resources.WorkingSet([])

ws.add_entry(path)

dist = ws.by_key.get(DIST_NAME)

if dist is None:

# We didn't find an egg-info/dist-info in the given path, but if a

# setup.py exists we can generate it

setup_py = os.path.join(path, 'setup.py')

if os.path.isfile(setup_py):

with _silence():

run_setup(os.path.join(path, 'setup.py'),

['egg_info'])

for dist in pkg_resources.find_distributions(path, True):

# There should be only one...

return dist

return dist

def _do_download(self, version='', find_links=None):

if find_links:

allow_hosts = ''

index_url = None

else:

allow_hosts = None

index_url = self.index_url

# Annoyingly, setuptools will not handle other arguments to

# Distribution (such as options) before handling setup_requires, so it

# is not straightfoward to programmatically augment the arguments which

# are passed to easy_install

class _Distribution(Distribution):

def get_option_dict(self, command_name):

opts = Distribution.get_option_dict(self, command_name)

if command_name == 'easy_install':

if find_links is not None:

opts['find_links'] = ('setup script', find_links)

if index_url is not None:

opts['index_url'] = ('setup script', index_url)

if allow_hosts is not None:

opts['allow_hosts'] = ('setup script', allow_hosts)

return opts

if version:

req = '{0}=={1}'.format(DIST_NAME, version)

else:

req = DIST_NAME

attrs = {'setup_requires': [req]}

try:

if DEBUG:

_Distribution(attrs=attrs)

else:

with _silence():

_Distribution(attrs=attrs)

# If the setup_requires succeeded it will have added the new dist to

# the main working_set

return pkg_resources.working_set.by_key.get(DIST_NAME)

except Exception as e:

if DEBUG:

raise

msg = 'Error retrieving {0} from {1}:\n{2}'

if find_links:

source = find_links[0]

elif index_url != INDEX_URL:

source = index_url

else:

source = 'PyPI'

raise Exception(msg.format(DIST_NAME, source, repr(e)))

def _do_upgrade(self, dist):

# Build up a requirement for a higher bugfix release but a lower minor

# release (so API compatibility is guaranteed)

# sketchy version parsing--maybe come up with something a bit more

# robust for this

major, minor = (int(part) for part in dist.parsed_version[:2])

next_minor = '.'.join([str(major), str(minor + 1), '0'])

req = pkg_resources.Requirement.parse(

'{0}>{1},<{2}'.format(DIST_NAME, dist.version, next_minor))

package_index = PackageIndex(index_url=self.index_url)

upgrade = package_index.obtain(req)

if upgrade is not None:

return self._do_download(version=upgrade.version)

def _check_submodule(self):

"""

Check if the given path is a git submodule.

See the docstrings for ``_check_submodule_using_git`` and

``_check_submodule_no_git`` for futher details.

"""

if (self.path is None or

(os.path.exists(self.path) and not os.path.isdir(self.path))):

return False

if self.use_git:

return self._check_submodule_using_git()

else:

return self._check_submodule_no_git()

def _check_submodule_using_git(self):

"""

Check if the given path is a git submodule. If so, attempt to initialize

and/or update the submodule if needed.

This function makes calls to the ``git`` command in subprocesses. The

``_check_submodule_no_git`` option uses pure Python to check if the given

path looks like a git submodule, but it cannot perform updates.

"""

cmd = ['git', 'submodule', 'status', '--', self.path]

try:

log.info('Running `{0}`; use the --no-git option to disable git '

'commands'.format(' '.join(cmd)))

returncode, stdout, stderr = run_cmd(cmd)

except _CommandNotFound:

# The git command simply wasn't found; this is most likely the

# case on user systems that don't have git and are simply

# trying to install the package from PyPI or a source

# distribution. Silently ignore this case and simply don't try

# to use submodules

return False

stderr = stderr.strip()

if returncode != 0 and stderr:

# Unfortunately the return code alone cannot be relied on, as

# earlier versions of git returned 0 even if the requested submodule

# does not exist

# This is a warning that occurs in perl (from running git submodule)

# which only occurs with a malformatted locale setting which can

# happen sometimes on OSX. See again

# https://github.com/astropy/astropy/issues/2749

perl_warning = ('perl: warning: Falling back to the standard locale '

'("C").')

if not stderr.strip().endswith(perl_warning):

# Some other uknown error condition occurred

log.warn('git submodule command failed '

'unexpectedly:\n{0}'.format(stderr))

return False

# Output of `git submodule status` is as follows:

#

# 1: Status indicator: '-' for submodule is uninitialized, '+' if

# submodule is initialized but is not at the commit currently indicated

# in .gitmodules (and thus needs to be updated), or 'U' if the

# submodule is in an unstable state (i.e. has merge conflicts)

#

# 2. SHA-1 hash of the current commit of the submodule (we don't really

# need this information but it's useful for checking that the output is

# correct)

#

# 3. The output of `git describe` for the submodule's current commit

# hash (this includes for example what branches the commit is on) but

# only if the submodule is initialized. We ignore this information for

# now

_git_submodule_status_re = re.compile(

'^(?P<status>[+-U ])(?P<commit>[0-9a-f]{40}) '

'(?P<submodule>\S+)( .*)?$')

# The stdout should only contain one line--the status of the

# requested submodule

m = _git_submodule_status_re.match(stdout)

if m:

# Yes, the path *is* a git submodule

self._update_submodule(m.group('submodule'), m.group('status'))

return True

else:

log.warn(

'Unexpected output from `git submodule status`:\n{0}\n'

'Will attempt import from {1!r} regardless.'.format(

stdout, self.path))

return False

def _check_submodule_no_git(self):

"""

Like ``_check_submodule_using_git``, but simply parses the .gitmodules file

to determine if the supplied path is a git submodule, and does not exec any

subprocesses.

This can only determine if a path is a submodule--it does not perform

updates, etc. This function may need to be updated if the format of the

.gitmodules file is changed between git versions.

"""

gitmodules_path = os.path.abspath('.gitmodules')

if not os.path.isfile(gitmodules_path):

return False

# This is a minimal reader for gitconfig-style files. It handles a few of

# the quirks that make gitconfig files incompatible with ConfigParser-style

# files, but does not support the full gitconfig syntaix (just enough

# needed to read a .gitmodules file).

gitmodules_fileobj = io.StringIO()

# Must use io.open for cross-Python-compatible behavior wrt unicode

with io.open(gitmodules_path) as f:

for line in f:

# gitconfig files are more flexible with leading whitespace; just

# go ahead and remove it

line = line.lstrip()

# comments can start with either # or ;

if line and line[0] in (':', ';'):

continue

gitmodules_fileobj.write(line)

gitmodules_fileobj.seek(0)

cfg = RawConfigParser()

try:

cfg.readfp(gitmodules_fileobj)

except Exception as exc:

log.warn('Malformatted .gitmodules file: {0}\n'

'{1} cannot be assumed to be a git submodule.'.format(

exc, self.path))

return False

for section in cfg.sections():

if not cfg.has_option(section, 'path'):

continue

submodule_path = cfg.get(section, 'path').rstrip(os.sep)

if submodule_path == self.path.rstrip(os.sep):

return True

return False

def _update_submodule(self, submodule, status):

if status == ' ':

# The submodule is up to date; no action necessary

return

elif status == '-':

if self.offline:

raise _AHBootstrapSystemExit(

"Cannot initialize the {0} submodule in --offline mode; "

"this requires being able to clone the submodule from an "

"online repository.".format(submodule))

cmd = ['update', '--init']

action = 'Initializing'

elif status == '+':

cmd = ['update']

action = 'Updating'

if self.offline:

cmd.append('--no-fetch')

elif status == 'U':

raise _AHBoostrapSystemExit(

'Error: Submodule {0} contains unresolved merge conflicts. '

'Please complete or abandon any changes in the submodule so that '

'it is in a usable state, then try again.'.format(submodule))

else:

log.warn('Unknown status {0!r} for git submodule {1!r}. Will '

'attempt to use the submodule as-is, but try to ensure '

'that the submodule is in a clean state and contains no '

'conflicts or errors.\n{2}'.format(status, submodule,

_err_help_msg))

return

err_msg = None

cmd = ['git', 'submodule'] + cmd + ['--', submodule]

log.warn('{0} {1} submodule with: `{2}`'.format(

action, submodule, ' '.join(cmd)))

try:

log.info('Running `{0}`; use the --no-git option to disable git '

'commands'.format(' '.join(cmd)))

returncode, stdout, stderr = run_cmd(cmd)

except OSError as e:

err_msg = str(e)

else:

if returncode != 0:

err_msg = stderr

if err_msg is not None:

log.warn('An unexpected error occurred updating the git submodule '

'{0!r}:\n{1}\n{2}'.format(submodule, err_msg,

"""

An exception raised when a command run with run_cmd is not found on the

system.

"""

Run a command in a subprocess, given as a list of command-line

arguments.

Returns a ``(returncode, stdout, stderr)`` tuple.

"""

try:

p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)

# XXX: May block if either stdout or stderr fill their buffers;

# however for the commands this is currently used for that is

# unlikely (they should have very brief output)

stdout, stderr = p.communicate()

except OSError as e:

if DEBUG:

raise

if e.errno == errno.ENOENT:

msg = 'Command not found: `{0}`'.format(' '.join(cmd))

raise _CommandNotFound(msg, cmd)

else:

raise _AHBoostrapSystemExit(

'An unexpected error occurred when running the '

'`{0}` command:\n{1}'.format(' '.join(cmd), str(e)))

# Can fail of the default locale is not configured properly. See

# https://github.com/astropy/astropy/issues/2749. For the purposes under

# consideration 'latin1' is an acceptable fallback.

try:

stdio_encoding = locale.getdefaultlocale()[1] or 'latin1'

except ValueError:

# Due to an OSX oddity locale.getdefaultlocale() can also crash

# depending on the user's locale/language settings. See:

# http://bugs.python.org/issue18378

stdio_encoding = 'latin1'

# Unlikely to fail at this point but even then let's be flexible

if not isinstance(stdout, _text_type):

stdout = stdout.decode(stdio_encoding, 'replace')

if not isinstance(stderr, _text_type):

stderr = stderr.decode(stdio_encoding, 'replace')

"""A noop writeable object."""

errors = '' # Required for Python 3.x

encoding = 'utf-8'

def write(self, s):

pass

def flush(self):

"""A context manager that silences sys.stdout and sys.stderr."""

old_stdout = sys.stdout

old_stderr = sys.stderr

sys.stdout = _DummyFile()

sys.stderr = _DummyFile()

exception_occurred = False

try:

yield

except:

exception_occurred = True

# Go ahead and clean up so that exception handling can work normally

sys.stdout = old_stdout

sys.stderr = old_stderr

raise

if not exception_occurred:

sys.stdout = old_stdout

If the problem persists consider installing astropy_helpers manually using pip

(`pip install astropy_helpers`) or by manually downloading the source archive,

extracting it, and installing by running `python setup.py install` from the

root of the extracted source code.

def __init__(self, *args):

if not args:

msg = 'An unknown problem occurred bootstrapping astropy_helpers.'

else:

msg = args[0]

msg += '\n' + _err_help_msg

def __getattr__(self, attr):

return getattr(distutils.log, attr)

def warn(self, msg, *args):

self._log_to_stderr(distutils.log.WARN, msg, *args)

def error(self, msg):

self._log_to_stderr(distutils.log.ERROR, msg, *args)

def fatal(self, msg):

self._log_to_stderr(distutils.log.FATAL, msg, *args)

def log(self, level, msg, *args):

if level in (distutils.log.WARN, distutils.log.ERROR,

distutils.log.FATAL):

self._log_to_stderr(level, msg, *args)

else:

distutils.log.log(level, msg, *args)

def _log_to_stderr(self, level, msg, *args):

# This is the only truly 'public' way to get the current threshold

# of the log

current_threshold = distutils.log.set_threshold(distutils.log.WARN)

distutils.log.set_threshold(current_threshold)

if level >= current_threshold:

if args:

msg = msg % args

sys.stderr.write('%s\n' % msg)

"""

Ensure that the `astropy_helpers` module is available and is importable.

This supports automatic submodule initialization if astropy_helpers is

included in a project as a git submodule, or will download it from PyPI if

necessary.

Parameters

----------

path : str or None, optional

A filesystem path relative to the root of the project's source code

that should be added to `sys.path` so that `astropy_helpers` can be

imported from that path.

If the path is a git submodule it will automatically be initialzed

and/or updated.

The path may also be to a ``.tar.gz`` archive of the astropy_helpers

source distribution. In this case the archive is automatically

unpacked and made temporarily available on `sys.path` as a ``.egg``

archive.

If `None` skip straight to downloading.

download_if_needed : bool, optional

If the provided filesystem path is not found an attempt will be made to

download astropy_helpers from PyPI. It will then be made temporarily

available on `sys.path` as a ``.egg`` archive (using the

``setup_requires`` feature of setuptools. If the ``--offline`` option

is given at the command line the value of this argument is overridden

to `False`.

index_url : str, optional

If provided, use a different URL for the Python package index than the

main PyPI server.

use_git : bool, optional

If `False` no git commands will be used--this effectively disables

support for git submodules. If the ``--no-git`` option is given at the

command line the value of this argument is overridden to `False`.

auto_upgrade : bool, optional

By default, when installing a package from a non-development source

distribution ah_boostrap will try to automatically check for patch

releases to astropy-helpers on PyPI and use the patched version over

any bundled versions. Setting this to `False` will disable that

functionality. If the ``--offline`` option is given at the command line

the value of this argument is overridden to `False`.

offline : bool, optional

If `False` disable all actions that require an internet connection,

including downloading packages from the package index and fetching

updates to any git submodule. Defaults to `True`.

"""

global _config

config = _config.copy()

config.update(**kwargs)

bootstrapper = _Bootstrapper(**config)