def download_package(self, package):
assert type(package) is RepoPackage
sh.mkdirs(DOWNLOAD_CACHE)
auth = licensing.get_be_license_auth()
send_license = package.requires_be_license
license_installed = auth is not None
# A license is required for this package, but no license is installed
if not license_installed and send_license:
msg = '\n'.join([
wrapped('If you have purchased ActivePython Business Edition, '
'please login to your account at:'),
' https://account.activestate.com/',
wrapped('and download and run the license installer for your '
'platform.'),
'',
wrapped('Please visit <%s> to learn more about the '
'ActivePython Business Edition offering.' % \
licensing.BE_HOME_PAGE)])
raise error.PackageAccessError(
package, 'requires Business Edition subscription', msg)
try:
# At this point, the user is already known to have a BE license
file_location, _ = net.download_file(
package.download_url,
DOWNLOAD_CACHE,
dict(
auth=auth,
use_cache=True, # XXX: this introduces network delay
# (If-None-Match) despite having the file
# in download cache
# TODO: abstract client.store...autosync
save_properties=True,
start_info='{{status}}: [{0}] {1} {2}'.format(
six.moves.urlparse(package.download_url).netloc,
package.name,
package.printable_version)),
interactive=self.pypmenv.options['interactive'])
except six.moves.HTTPError as e:
reason = str(e)
LOG.debug("HTTPError while accessing URL: %s -- reason: %s",
package.download_url, reason)
if send_license and e.code in (401, 402, 403):
msg = wrapped(
'Your ActivePython Business Edition subscription seems to '
'have expired. Please visit your account at '
'https://account.activestate.com/ to renew your subscription.'
)
else:
msg = ''
raise error.PackageAccessError(package, reason, msg)
return file_location
def sync_repository(self, remote_repository, force, verbose=True, interactive=True):
"""Sync the cache for a remote repository"""
with dlocked(self.path):
assert isinstance(remote_repository, RemoteRepository)
idx_path = self.get_local_index_path(remote_repository)
sh.mkdirs(dirname(idx_path))
return remote_repository.download_index(
idx_path, force, verbose, interactive=interactive)
def get_repository(self, pyver, osarch, autocreate=False):
path = xjoin(self.path, pyver, osarch)
url = '/'.join([self.url, pyver, osarch])
if autocreate:
# create, if does not already exist
sh.mkdirs(path)
return Repository(path, self.name, pyver, osarch, url)
def get_repository(self, pyver, osarch, autocreate=False):
path = xjoin(self.path, pyver, osarch)
url = '/'.join([self.url, pyver, osarch])
if autocreate:
# create, if does not already exist
sh.mkdirs(path)
return Repository(path, self.name, pyver, osarch, url)
def download_package(self, package):
assert type(package) is RepoPackage
sh.mkdirs(DOWNLOAD_CACHE)
auth = licensing.get_be_license_auth()
send_license = package.requires_be_license
license_installed = auth is not None
# A license is required for this package, but no license is installed
if not license_installed and send_license:
msg = '\n'.join([
wrapped('If you have purchased ActivePython Business Edition, '
'please login to your account at:'),
' https://account.activestate.com/',
wrapped('and download and run the license installer for your '
'platform.'),
'',
wrapped('Please visit <%s> to learn more about the '
'ActivePython Business Edition offering.' % \
licensing.BE_HOME_PAGE)])
raise error.PackageAccessError(
package, 'requires Business Edition subscription', msg)
try:
# At this point, the user is already known to have a BE license
file_location, _ = net.download_file(
package.download_url,
DOWNLOAD_CACHE,
dict(
auth=auth,
use_cache=True, # XXX: this introduces network delay
# (If-None-Match) despite having the file
# in download cache
# TODO: abstract client.store...autosync
save_properties=True,
start_info='{{status}}: [{0}] {1} {2}'.format(
six.moves.urlparse(package.download_url).netloc,
package.name, package.printable_version)),
interactive=self.pypmenv.options['interactive'])
except six.moves.HTTPError as e:
reason = str(e)
LOG.debug("HTTPError while accessing URL: %s -- reason: %s",
package.download_url, reason)
if send_license and e.code in (401, 402, 403):
msg = wrapped(
'Your ActivePython Business Edition subscription seems to '
'have expired. Please visit your account at '
'https://account.activestate.com/ to renew your subscription.'
)
else:
msg = ''
raise error.PackageAccessError(package, reason, msg)
return file_location
def test_sh_rm_dir():
with sh.tmpdir():
sh.mkdirs('adir')
with sh.cd('adir'):
with open('afile', 'w') as f: f.close()
assert path.exists('afile')
assert path.exists('adir')
sh.rm('adir')
assert not path.exists('adir')
def ensure_write_access(self):
if not os.path.exists(self.user_base_dir):
# user may not have a user base directory *yet*
sh.mkdirs(self.user_base_dir)
# This is unlikely to happen
if not os.access(self.user_base_dir, os.W_OK):
raise NoWriteAccess(
'no write access to user base "%s"' % self.user_base_dir)
def install_remote_file(pypmenv, url):
"""Install a .pypm file from an URL
Dependency is not automatically resolved, although we should do this in
future
"""
sh.mkdirs(DOWNLOAD_CACHE)
local_file, _ = net.download_file(url, DOWNLOAD_CACHE)
return install_local_file(pypmenv, local_file)
def ensure_write_access(self):
if not os.path.exists(self.user_base_dir):
# user may not have a user base directory *yet*
sh.mkdirs(self.user_base_dir)
# This is unlikely to happen
if not os.access(self.user_base_dir, os.W_OK):
raise NoWriteAccess('no write access to user base "%s"' %
self.user_base_dir)
def download_package(self, package):
assert type(package) is RepoPackage
sh.mkdirs(DOWNLOAD_CACHE)
LOG.info('Get: [%s] %s %s',
six.moves.urlparse(package.download_url).netloc,
package.name,
package.printable_version)
auth = licensing.get_be_license_auth()
send_license = package.requires_be_license
license_installed = auth is not None
# A license is required for this package, but no license is installed
if not license_installed and send_license:
msg = '\n'.join([
wrapped('If you have purchased ActivePython Business Edition, '
'please login to your account at:'),
' https://account.activestate.com/',
wrapped('and download and run the license installer for your '
'platform.'),
'',
wrapped('Please visit <%s> to learn more about the '
'ActivePython Business Edition offering.' % \
licensing.BE_HOME_PAGE)])
raise error.PackageAccessError(
package, 'requires Business Edition subscription', msg)
try:
# At this point, the user is already known to have a BE license
file_location = net.download_file(
package.download_url,
DOWNLOAD_CACHE,
dict(auth=auth),
interactive=self.pypmenv.options['interactive'])
except six.moves.HTTPError as e:
reason = str(e)
LOG.debug("HTTPError while accessing URL: %s -- reason: %s",
package.download_url, reason)
if send_license and e.code in (401, 402, 403):
msg = wrapped(
'Your ActivePython Business Edition subscription seems to '
'have expired. Please visit your account at '
'https://account.activestate.com/ to renew your subscription.'
)
else:
msg = ''
raise error.PackageAccessError(package, reason, msg)
return file_location
def _pep370_fix_path_unix():
"""If ~/.local/bin is not in $PATH, automatically add them
Do this only with the user's consent. And do not run this check more than
once (i.e., run only when PyPM is *first run*).
"""
if sys.platform.startswith('win'):
return # MSI does this on Windows
# Proceed only when the terminal is interactive and was never run before
isatty = (sys.stdin.isatty() and sys.stdout.isatty())
firstrun_file = P.join(application.locations.user_cache_dir,
'.firstrun-pep370')
if (not isatty) or P.exists(firstrun_file):
return
import site
from datetime import datetime
pathenv = [
P.abspath(x.strip()) for x in os.environ.get('PATH', '').split(':')
]
binpath = P.abspath(P.join(site.USER_BASE, 'bin'))
profile = P.expanduser('~/.profile' if sys.platform ==
'darwin' else '~/.bashrc')
profile_lines = [
'# PEP 370 PATH added by PyPM on %s' % datetime.now(),
'export PATH=%s:$PATH' % binpath,
]
already_in_profile = P.exists(profile) and profile_lines[1] in [
l.strip() for l in open(profile).readlines()
]
# Proceed only if ~/.local/bin is neither in $PATH, nor added to profile
if binpath in pathenv or already_in_profile:
return
# Add to profile on the user's consent
msg = ('Packages will install their script files to "%s" '
'(as per PEP 370). This directory is not yet in your $PATH. '
'Would you like PyPM to add it by appending to "%s"?') % (
concise_path(binpath), concise_path(profile))
if textui.askyesno(wrapped(msg, '*** '), default=True):
if P.exists(profile):
sh.cp(profile, profile + '.bak') # take a backup first
with open(profile, 'a') as f:
f.write('\n%s\n' % '\n'.join(profile_lines))
print('You may now reopen your shell for the changes to take effect.')
sh.mkdirs(P.dirname(firstrun_file))
with open(firstrun_file, 'w') as f:
pass # prevent future runs
def setup_trace(l, tracefile):
"""Trace logging calls to a standard log file
Log file name and location will be determined based on platform.
"""
l.setLevel(logging.DEBUG) # trace file must have >=DEBUG entries
sh.mkdirs(dirname(tracefile))
_rollover_log(tracefile)
_begin_log_section(tracefile)
h = logging.FileHandler(tracefile)
h.setFormatter(
logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
l.addHandler(h)
def sync_repository(self,
remote_repository,
force,
verbose=True,
interactive=True):
"""Sync the cache for a remote repository"""
with dlocked(self.path):
assert isinstance(remote_repository, RemoteRepository)
idx_path = self.get_local_index_path(remote_repository)
sh.mkdirs(P.dirname(idx_path))
return remote_repository.download_index(idx_path,
force,
verbose,
interactive=interactive)
def setup_trace(l, tracefile):
"""Trace logging calls to a standard log file
Log file name and location will be determined based on platform.
"""
l.setLevel(logging.DEBUG) # trace file must have >=DEBUG entries
sh.mkdirs(dirname(tracefile))
_rollover_log(tracefile)
_begin_log_section(tracefile)
h = logging.FileHandler(tracefile)
h.setFormatter(
logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
l.addHandler(h)
def _pep370_fix_path_unix():
"""If ~/.local/bin is not in $PATH, automatically add them
Do this only with the user's consent. And do not run this check more than
once (i.e., run only when PyPM is *first run*).
"""
if sys.platform.startswith('win'):
return # MSI does this on Windows
# Proceed only when the terminal is interactive and was never run before
isatty = (sys.stdin.isatty() and sys.stdout.isatty())
firstrun_file = P.join(application.locations.user_cache_dir, '.firstrun-pep370')
if (not isatty) or P.exists(firstrun_file):
return
import site
from datetime import datetime
pathenv = [P.abspath(x.strip()) for x in os.environ.get('PATH', '').split(':')]
binpath = P.abspath(P.join(site.USER_BASE, 'bin'))
profile = P.expanduser('~/.profile' if sys.platform == 'darwin' else '~/.bashrc')
profile_lines = [
'# PEP 370 PATH added by PyPM on %s' % datetime.now(),
'export PATH=%s:$PATH' % binpath,
]
already_in_profile = P.exists(profile) and profile_lines[1] in [
l.strip() for l in open(profile).readlines()
]
# Proceed only if ~/.local/bin is neither in $PATH, nor added to profile
if binpath in pathenv or already_in_profile:
return
# Add to profile on the user's consent
msg = (
'Packages install script files to "%s" '
'(as per PEP 370). This directory is not yet in your $PATH. '
'Would you like PyPM to add it to $PATH by modifying (with backup) "%s"?'
) % (binpath, profile)
if textui.askyesno(wrapped(msg, '*** '), default=True):
if P.exists(profile):
sh.cp(profile, profile+'.bak') # take a backup first
with open(profile, 'a') as f:
f.write('\n%s\n' % '\n'.join(profile_lines))
print ('You may now reopen your shell for the changes to take effect.')
sh.mkdirs(P.dirname(firstrun_file))
with open(firstrun_file, 'w') as f: pass # prevent future runs
def do_copy_custom(self, subcmd, opts, *paths):
"""${cmd_name}: Copy packages into the appropriate repository
Use this command to *manually* copy the *custom* PyPM packages that
won't be built by the *automated* pypm-builder/grail. This includes the
following cases,
1. "extra" PyPM packages available in the "GoldBits" directory of
ActivePython (eg: as.openssl)
2. Proprietary packages (eg: pyimsl from VNI)
Use ``MultiRepositoryConfig`` (etc/activestate.conf) to configure how to
allocate the custom packages, i.e., where to put them in "free" or "be"
repo.
Example::
$ bin/pypm-repository -c etc/activestate.conf copy_custom \
$NAS/ActivePython/2.7.0.2/GoldBits/internal/extra/*.pypm
${cmd_usage}
${cmd_option_list}
"""
with self.bootstrapped():
mreposet = MultiRepositorySet(
self.options.multi_repository_set_path,
self.options.configfile
)
for path in paths:
bpkg = PackageFile(path).to_binary_package()
repo = mreposet.get_repository(bpkg)
target_path = P.join(
repo.path, 'pool', bpkg.name[0], bpkg.name[:2],
os.path.basename(path))
sh.mkdirs(P.dirname(target_path))
action = 'OVERWRITE' if P.exists(target_path) else 'CREATE'
LOG.info('%s %s %s', action, repo.name, target_path)
if not opts.dry_run:
if P.exists(target_path) and not opts.force:
raise IOError('cannot overwrite: %s' % target_path)
sh.cp(path, target_path)
repo.uptree.mark_dirty(target_path)
def __init__(self, pyenv, repository_list, **options):
"""
pyenv -- An instance of pypm.common.python.*Environment
repository_list -- List of repositories
options -- Override for DEFAULT_OPTIONS
"""
self.pyenv = pyenv
self.repository_list = repository_list
self.options = DEFAULT_OPTIONS.copy()
self.options.update(options)
self.pypm_dir = P.join(self.pyenv.site_packages_dir, '_pypm')
self.repo_store = store.RepoPackageStore(
RemoteRepositoryManager(IDX_PATH), repository_list)
if not P.exists(self.pypm_dir):
self.pyenv.ensure_write_access()
sh.mkdirs(self.pypm_dir)
self.installed_store = store.InstalledPackageStore(
P.join(self.pypm_dir, 'installed.db'))
def do_copy_custom(self, subcmd, opts, *paths):
"""${cmd_name}: Copy packages into the appropriate repository
Use this command to *manually* copy the *custom* PyPM packages that
won't be built by the *automated* pypm-builder/grail. This includes the
following cases,
1. "extra" PyPM packages available in the "GoldBits" directory of
ActivePython (eg: as.openssl)
2. Proprietary packages (eg: pyimsl from VNI)
Use ``MultiRepositoryConfig`` (etc/activestate.conf) to configure how to
allocate the custom packages, i.e., where to put them in "free" or "be"
repo.
Example::
$ bin/pypm-repository -c etc/activestate.conf copy_custom \
$NAS/ActivePython/2.7.0.2/GoldBits/internal/extra/*.pypm
${cmd_usage}
${cmd_option_list}
"""
with self.bootstrapped():
mreposet = MultiRepositorySet(
self.options.multi_repository_set_path,
self.options.configfile)
for path in paths:
bpkg = PackageFile(path).to_binary_package()
repo = mreposet.get_repository(bpkg)
target_path = P.join(repo.path, 'pool', bpkg.name[0],
bpkg.name[:2], os.path.basename(path))
sh.mkdirs(P.dirname(target_path))
action = 'OVERWRITE' if P.exists(target_path) else 'CREATE'
LOG.info('%s %s %s', action, repo.name, target_path)
if not opts.dry_run:
if P.exists(target_path) and not opts.force:
raise IOError('cannot overwrite: %s' % target_path)
sh.cp(path, target_path)
repo.uptree.mark_dirty(target_path)
def __init__(self, pyenv, repository_list, **options):
"""
pyenv -- An instance of pypm.common.python.*Environment
repository_list -- List of repositories
options -- Override for DEFAULT_OPTIONS
"""
self.pyenv = pyenv
self.repository_list = repository_list
self.options = DEFAULT_OPTIONS.copy()
self.options.update(options)
self.pypm_dir = P.join(self.pyenv.site_packages_dir, '_pypm')
self.repo_store = store.RepoPackageStore(
RemoteRepositoryManager(IDX_PATH), repository_list)
if not P.exists(self.pypm_dir):
self.pyenv.ensure_write_access()
sh.mkdirs(self.pypm_dir)
self.installed_store = store.InstalledPackageStore(
P.join(self.pypm_dir, 'installed.db')
)
def handledby(l, filename, create_dirs=False, level=None, formatter=None):
"""Momentarily handle logger `l` using FileHandler
Within the 'with' context, all logging calls made to the logger `l` will get
written to the file `filename`. When exiting the 'with' context, this file
is closed and the handler will be removed.
"""
assert isabs(filename), 'not an absolute path: {0}'.format(filename)
if create_dirs:
sh.mkdirs(dirname(filename))
h = logging.FileHandler(filename)
if level:
h.setLevel(level)
if formatter:
h.setFormatter(formatter)
l.addHandler(h)
try:
yield
finally:
h.close()
l.removeHandler(h)
def handledby(l, filename, create_dirs=False, level=None, formatter=None):
"""Momentarily handle logger `l` using FileHandler
Within the 'with' context, all logging calls made to the logger `l` will get
written to the file `filename`. When exiting the 'with' context, this file
is closed and the handler will be removed.
"""
assert isabs(filename), 'not an absolute path: {0}'.format(filename)
if create_dirs:
sh.mkdirs(dirname(filename))
h = logging.FileHandler(filename)
if level:
h.setLevel(level)
if formatter:
h.setFormatter(formatter)
l.addHandler(h)
try:
yield
finally:
h.close()
l.removeHandler(h)
def __init__(self, path):
# local cache directory where repository indexes will be stored
self.path = path
sh.mkdirs(path)
def __init__(self, path):
# local cache directory where repository indexes will be stored
self.path = path
sh.mkdirs(path)
def create_repositories(self):
"""Create repositories for supported configuration"""
for osarch in supported.os_architectures:
for pyver in supported.py_versions:
sh.mkdirs(join(self.path, pyver, osarch))
def create_repositories(self):
"""Create repositories for supported configuration"""
for osarch in supported.os_architectures:
for pyver in supported.py_versions:
sh.mkdirs(P.join(self.path, pyver, osarch))
