def test_setup_requires_overrides_version_conflict(self):
"""
Regression test for issue #323.
Ensures that a distribution's setup_requires requirements can still be
installed and used locally even if a conflicting version of that
requirement is already on the path.
"""
pr_state = pkg_resources.__getstate__()
fake_dist = PRDistribution('does-not-matter', project_name='foobar',
version='0.0')
working_set.add(fake_dist)
try:
with contexts.tempdir() as temp_dir:
test_pkg = create_setup_requires_package(temp_dir)
test_setup_py = os.path.join(test_pkg, 'setup.py')
with contexts.quiet() as (stdout, stderr):
# Don't even need to install the package, just
# running the setup.py at all is sufficient
run_setup(test_setup_py, ['--name'])
lines = stdout.readlines()
assert len(lines) > 0
assert lines[-1].strip(), 'test_pkg'
finally:
pkg_resources.__setstate__(pr_state)
def test_setup_requires_overrides_version_conflict(self, use_setup_cfg):
"""
Regression test for distribution issue 323:
https://bitbucket.org/tarek/distribute/issues/323
Ensures that a distribution's setup_requires requirements can still be
installed and used locally even if a conflicting version of that
requirement is already on the path.
"""
fake_dist = PRDistribution('does-not-matter', project_name='foobar',
version='0.0')
working_set.add(fake_dist)
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
test_pkg = create_setup_requires_package(
temp_dir, use_setup_cfg=use_setup_cfg)
test_setup_py = os.path.join(test_pkg, 'setup.py')
with contexts.quiet() as (stdout, stderr):
# Don't even need to install the package, just
# running the setup.py at all is sufficient
run_setup(test_setup_py, [str('--name')])
lines = stdout.readlines()
assert len(lines) > 0
assert lines[-1].strip() == 'test_pkg'
def test_setup_requires_with_attr_version(self, use_setup_cfg):
def make_dependency_sdist(dist_path, distname, version):
files = [(
'setup.py',
DALS("""
import setuptools
setuptools.setup(
name={name!r},
version={version!r},
py_modules=[{name!r}],
)
""".format(name=distname, version=version)),
), (
distname + '.py',
DALS("""
version = 42
"""),
)]
make_sdist(dist_path, files)
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
test_pkg = create_setup_requires_package(
temp_dir, setup_attrs=dict(version='attr: foobar.version'),
make_package=make_dependency_sdist,
use_setup_cfg=use_setup_cfg+('version',),
)
test_setup_py = os.path.join(test_pkg, 'setup.py')
with contexts.quiet() as (stdout, stderr):
run_setup(test_setup_py, [str('--version')])
lines = stdout.readlines()
assert len(lines) > 0
assert lines[-1].strip() == '42'
def _directory_import(path):
"""
Import astropy_helpers from the given path, which will be added to
sys.path.
Must return True if the import succeeded, and False otherwise.
"""
# Return True on success, False on failure but download is allowed, and
# otherwise raise SystemExit
path = os.path.abspath(path)
# Use an empty WorkingSet rather than the man pkg_resources.working_set,
# since on older versions of setuptools this will invoke a VersionConflict
# when trying to install an upgrade
ws = pkg_resources.WorkingSet([])
ws.add_entry(path)
dist = ws.by_key.get(DIST_NAME)
if dist is None:
# We didn't find an egg-info/dist-info in the given path, but if a
# setup.py exists we can generate it
setup_py = os.path.join(path, 'setup.py')
if os.path.isfile(setup_py):
with _silence():
run_setup(os.path.join(path, 'setup.py'), ['egg_info'])
for dist in pkg_resources.find_distributions(path, True):
# There should be only one...
return dist
return dist
def test_bootstrap_from_directory(tmpdir, testpackage, capsys):
"""
Tests simply bundling a copy of the astropy_helpers source code in its
entirety bundled directly in the source package and not in an archive.
"""
import ah_bootstrap
source = tmpdir.mkdir('source')
testpackage.copy(source.join('_astropy_helpers_test_'))
with source.as_cwd():
source.join('setup.py').write(TEST_SETUP_PY.format(args=''))
run_setup('setup.py', [])
stdout, stderr = capsys.readouterr()
stdout = stdout.splitlines()
if stdout:
path = stdout[-1].strip()
else:
path = ''
# Ensure that the astropy_helpers used by the setup.py is the one that
# was imported from git submodule
assert path == str(source.join('_astropy_helpers_test_',
'_astropy_helpers_test_',
'__init__.py'))
def _directory_import(path):
"""
Import astropy_helpers from the given path, which will be added to
sys.path.
Must return True if the import succeeded, and False otherwise.
"""
# Return True on success, False on failure but download is allowed, and
# otherwise raise SystemExit
path = os.path.abspath(path)
pkg_resources.working_set.add_entry(path)
dist = pkg_resources.working_set.by_key.get(DIST_NAME)
if dist is None:
# We didn't find an egg-info/dist-info in the given path, but if a
# setup.py exists we can generate it
setup_py = os.path.join(path, 'setup.py')
if os.path.isfile(setup_py):
with _silence():
run_setup(os.path.join(path, 'setup.py'), ['egg_info'])
for dist in pkg_resources.find_distributions(path, True):
# There should be only one...
pkg_resources.working_set.add(dist, path, False)
break
return dist
def test_setup_requires_override_nspkg(self, use_setup_cfg):
"""
Like ``test_setup_requires_overrides_version_conflict`` but where the
``setup_requires`` package is part of a namespace package that has
*already* been imported.
"""
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
foobar_1_archive = os.path.join(temp_dir, 'foo.bar-0.1.tar.gz')
make_nspkg_sdist(foobar_1_archive, 'foo.bar', '0.1')
# Now actually go ahead an extract to the temp dir and add the
# extracted path to sys.path so foo.bar v0.1 is importable
foobar_1_dir = os.path.join(temp_dir, 'foo.bar-0.1')
os.mkdir(foobar_1_dir)
with tarfile.open(foobar_1_archive) as tf:
tf.extractall(foobar_1_dir)
sys.path.insert(1, foobar_1_dir)
dist = PRDistribution(foobar_1_dir, project_name='foo.bar',
version='0.1')
working_set.add(dist)
template = DALS("""\
import foo # Even with foo imported first the
# setup_requires package should override
import setuptools
setuptools.setup(**%r)
if not (hasattr(foo, '__path__') and
len(foo.__path__) == 2):
print('FAIL')
if 'foo.bar-0.2' not in foo.__path__[0]:
print('FAIL')
""")
test_pkg = create_setup_requires_package(
temp_dir, 'foo.bar', '0.2', make_nspkg_sdist, template,
use_setup_cfg=use_setup_cfg)
test_setup_py = os.path.join(test_pkg, 'setup.py')
with contexts.quiet() as (stdout, stderr):
try:
# Don't even need to install the package, just
# running the setup.py at all is sufficient
run_setup(test_setup_py, [str('--name')])
except pkg_resources.VersionConflict:
self.fail(
'Installing setup.py requirements '
'caused a VersionConflict')
assert 'FAIL' not in stdout.getvalue()
lines = stdout.readlines()
assert len(lines) > 0
assert lines[-1].strip() == 'test_pkg'
def test_download_if_needed(tmpdir, testpackage, capsys):
"""
Tests the case where astropy_helpers was not actually included in a
package, or is otherwise missing, and we need to "download" it.
This does not test actually downloading from the internet--this is normally
done through setuptools' easy_install command which can also install from a
source archive. From the point of view of ah_boostrap the two actions are
equivalent, so we can just as easily simulate this by providing a setup.cfg
giving the path to a source archive to "download" (as though it were a
URL).
"""
source = tmpdir.mkdir('source')
# Ensure ah_bootstrap is imported from the local directory
import ah_bootstrap
# Make a source distribution of the test package
with silence():
run_setup(str(testpackage.join('setup.py')),
['sdist', '--dist-dir=dist', '--formats=gztar'])
dist_dir = testpackage.join('dist')
with source.as_cwd():
source.join('setup.py').write(TEST_SETUP_PY.format(
args='download_if_needed=True'))
source.join('setup.cfg').write(textwrap.dedent("""\
[easy_install]
find_links = {find_links}
""".format(find_links=str(dist_dir))))
run_setup('setup.py', [])
stdout, stderr = capsys.readouterr()
# Just take the last line--on Python 2.6 distutils logs warning
# messages to stdout instead of stderr, causing them to be mixed up
# with our expected output
path = stdout.splitlines()[-1].strip()
# easy_install should have worked by 'installing' astropy_helpers as a
# .egg in the current directory
eggs = glob.glob('*.egg')
assert eggs
egg = source.join(eggs[0])
assert os.path.isdir(str(egg))
a = os.path.normcase(path)
b = os.path.normcase(str(egg.join('_astropy_helpers_test_',
'__init__.py')))
assert a == b
def archive_package(self, cache=True):
"""Downloads the source tree and makes the source distribution.
It yields triple of package name, filename of the source
distribution, and its full path. ::
with build.archive_package() as (package, filename, path):
sftp.put(path, filename)
:param cache: whether to cache the package file or not.
``True`` by default
:type cache: :class:`bool`
"""
logger_ = self.get_logger('archive_package')
with self.branch.fetch(self.commit.ref) as path:
setup_script = os.path.join(path, 'setup.py')
if not os.path.isfile(setup_script):
raise IOError('cannot found setup.py script in the source '
'tree {0!r}'.format(self.commit))
tag = '.{0}.{1:%Y%m%d%H%M%S}.{2!s:.7}'.format(
self.branch.label,
self.commit.committed_at.astimezone(UTC()),
self.commit
)
with capture_stdout() as buffer_:
run_setup(setup_script, ['--fullname'])
fullname = buffer_.getvalue().rstrip().splitlines()[-1]
package_name = fullname + tag
filename = package_name + '.tar.bz2'
if cache:
cache_dir_path = os.path.join(
tempfile.gettempdir(),
'asuka-dist-cache'
)
if not os.path.isdir(cache_dir_path):
os.makedirs(cache_dir_path)
cache_path = os.path.join(cache_dir_path, filename)
if os.path.isfile(cache_path):
logger_.info('cache exists: %s, skipping sdist...',
cache_path)
yield package_name, filename, cache_path
return
run_setup(setup_script, [
'egg_info', '--tag-build', tag,
'sdist', '--formats=bztar'
])
filepath = os.path.join(path, 'dist', filename)
logger_.info('sdist_path = %r', filepath)
if cache:
logger_.info('save sdist cache %s...', cache_path)
shutil.copyfile(filepath, cache_path)
yield package_name, filename, filepath
def test_setup_requires(self):
"""Regression test for Distribute issue #318
Ensure that a package with setup_requires can be installed when
setuptools is installed in the user site-packages without causing a
SandboxViolation.
"""
test_setup_attrs = {
'name': 'test_pkg', 'version': '0.0',
'setup_requires': ['foobar'],
'dependency_links': [os.path.abspath(self.dir)]
}
test_pkg = os.path.join(self.dir, 'test_pkg')
test_setup_py = os.path.join(test_pkg, 'setup.py')
os.mkdir(test_pkg)
f = open(test_setup_py, 'w')
f.write(textwrap.dedent("""\
import setuptools
setuptools.setup(**%r)
""" % test_setup_attrs))
f.close()
foobar_path = os.path.join(self.dir, 'foobar-0.1.tar.gz')
make_trivial_sdist(
foobar_path,
textwrap.dedent("""\
import setuptools
setuptools.setup(
name='foobar',
version='0.1'
)
"""))
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdout = StringIO()
sys.stderr = StringIO()
try:
try:
with reset_setup_stop_context():
run_setup(test_setup_py, ['install'])
except SandboxViolation:
self.fail('Installation caused SandboxViolation')
finally:
sys.stdout = old_stdout
sys.stderr = old_stderr
def test_bootstrap_from_submodule(tmpdir, testpackage, capsys):
"""
Tests importing _astropy_helpers_test_ from a submodule in a git
repository. This tests actually performing a fresh clone of the repository
without the submodule initialized, and that importing astropy_helpers in
that context works transparently after calling
`ah_boostrap.use_astropy_helpers`.
"""
orig_repo = tmpdir.mkdir('orig')
# Ensure ah_bootstrap is imported from the local directory
import ah_bootstrap
with orig_repo.as_cwd():
run_cmd('git', ['init'])
# Write a test setup.py that uses ah_bootstrap; it also ensures that
# any previous reference to astropy_helpers is first wiped from
# sys.modules
orig_repo.join('setup.py').write(TEST_SETUP_PY.format(args=''))
run_cmd('git', ['add', 'setup.py'])
# Add our own clone of the astropy_helpers repo as a submodule named
# astropy_helpers
run_cmd('git', ['submodule', 'add', str(testpackage),
'_astropy_helpers_test_'])
run_cmd('git', ['commit', '-m', 'test repository'])
os.chdir(str(tmpdir))
# Creates a clone of our test repo in the directory 'clone'
run_cmd('git', ['clone', 'orig', 'clone'])
os.chdir('clone')
run_setup('setup.py', [])
stdout, stderr = capsys.readouterr()
path = stdout.strip()
# Ensure that the astropy_helpers used by the setup.py is the one that
# was imported from git submodule
a = os.path.normcase(path)
b = os.path.normcase(str(tmpdir.join('clone', '_astropy_helpers_test_',
'_astropy_helpers_test_',
'__init__.py')))
assert a == b
def run_setup(self, setup_script, setup_base, args):
args = list(args)
if self.verbose>2:
v = 'v' * (self.verbose - 1)
args.insert(0,'-'+v)
elif self.verbose<2:
args.insert(0,'-q')
if self.dry_run:
args.insert(0,'-n')
log.info(
"Running %s %s", setup_script[len(setup_base)+1:], ' '.join(args)
)
try:
run_setup(setup_script, args)
except SystemExit, v:
raise DistutilsError("Setup script exited with %s" % (v.args[0],))
def run_setup(self, *args):
old_stdout = sys.stdout
old_stderr = sys.stderr
stdout = sys.stdout = StringIO()
stderr = sys.stderr = StringIO()
try:
run_setup('setup.py', args)
returncode = 0
except SystemExit as e:
returncode = e.args[0]
finally:
sys.stdout = old_stdout
sys.stderr = old_stderr
return (stdout.getvalue().strip(), stderr.getvalue().strip(),
returncode)
def test_update_git_devstr(package_template, capsys):
"""Tests that the commit number in the package's version string updates
after git commits even without re-running setup.py.
"""
run_setup('setup.py', ['--version'])
stdout, stderr = capsys.readouterr()
version = stdout.strip()
m = _DEV_VERSION_RE.match(version)
assert m, (
"Stdout did not match the version string pattern:"
"\n\n{0}\n\nStderr:\n\n{1}".format(stdout, stderr))
revcount = int(m.group(1))
import packagename
assert packagename.__version__ == version
# Make a silly git commit
with open('.test', 'w'):
pass
run_cmd('git', ['add', '.test'])
run_cmd('git', ['commit', '-m', 'test'])
import packagename.version
imp.reload(packagename.version)
# Previously this checked packagename.__version__, but in order for that to
# be updated we also have to re-import _astropy_init which could be tricky.
# Checking directly that the packagename.version module was updated is
# sufficient:
m = _DEV_VERSION_RE.match(packagename.version.version)
assert m
assert int(m.group(1)) == revcount + 1
# This doesn't test astropy_helpers.get_helpers.update_git_devstr directly
# since a copy of that function is made in packagename.version (so that it
# can work without astropy_helpers installed). In order to get test
# coverage on the actual astropy_helpers copy of that function just call it
# directly and compare to the value in packagename
from astropy_helpers.git_helpers import update_git_devstr
newversion = update_git_devstr(version, path=str(package_template))
assert newversion == packagename.version.version
def test_check_submodule_no_git(tmpdir, testpackage):
"""
Tests that when importing astropy_helpers from a submodule, it is still
recognized as a submodule even when using the --no-git option.
In particular this ensures that the auto-upgrade feature is not activated.
"""
orig_repo = tmpdir.mkdir('orig')
# Ensure ah_bootstrap is imported from the local directory
import ah_bootstrap
with orig_repo.as_cwd():
run_cmd('git', ['init'])
# Write a test setup.py that uses ah_bootstrap; it also ensures that
# any previous reference to astropy_helpers is first wiped from
# sys.modules
args = 'auto_upgrade=True'
orig_repo.join('setup.py').write(TEST_SETUP_PY.format(args=args))
run_cmd('git', ['add', 'setup.py'])
# Add our own clone of the astropy_helpers repo as a submodule named
# astropy_helpers
run_cmd('git', ['submodule', 'add', str(testpackage),
'_astropy_helpers_test_'])
run_cmd('git', ['commit', '-m', 'test repository'])
# Temporarily patch _do_upgrade to fail if called
class UpgradeError(Exception):
pass
def _do_upgrade(*args, **kwargs):
raise UpgradeError()
orig_do_upgrade = ah_bootstrap._do_upgrade
ah_bootstrap._do_upgrade = _do_upgrade
try:
run_setup('setup.py', ['--no-git'])
except UpgradeError:
pytest.fail('Attempted to run auto-upgrade despite importing '
'_astropy_helpers_test_ from a git submodule')
finally:
ah_bootstrap._do_upgrade = orig_do_upgrade
def test_setup_requires(self):
"""Regression test for Distribute issue #318
Ensure that a package with setup_requires can be installed when
setuptools is installed in the user site-packages without causing a
SandboxViolation.
"""
test_pkg = create_setup_requires_package(self.dir)
test_setup_py = os.path.join(test_pkg, 'setup.py')
try:
with quiet_context():
with reset_setup_stop_context():
run_setup(test_setup_py, ['install'])
except SandboxViolation:
self.fail('Installation caused SandboxViolation')
def _bundle_local_bentoml_if_installed_from_source(target_path):
"""
if bentoml is installed in editor mode(pip install -e), this will build a source
distribution with the local bentoml fork and add it to saved BentoService bundle
path under bundled_pip_dependencies directory
"""
# Find bentoml module path
(module_location,) = importlib.util.find_spec('bentoml').submodule_search_locations
bentoml_setup_py = os.path.abspath(os.path.join(module_location, '..', 'setup.py'))
# this is for BentoML developer to create BentoService containing custom develop
# branches of BentoML library, it is True only when BentoML module is installed in
# development mode via "pip install --editable ."
if not _is_pip_installed_bentoml() and os.path.isfile(bentoml_setup_py):
logger.info(
"Detected non-PyPI-released BentoML installed, copying local BentoML module"
"files to target saved bundle path.."
)
# Create tmp directory inside bentoml module for storing the bundled
# targz file. Since dist-dir can only be inside of the module directory
bundle_dir_name = '__bentoml_tmp_sdist_build'
source_dir = os.path.abspath(
os.path.join(module_location, '..', bundle_dir_name)
)
if os.path.isdir(source_dir):
shutil.rmtree(source_dir, ignore_errors=True)
os.mkdir(source_dir)
from setuptools import sandbox
sandbox.run_setup(
bentoml_setup_py,
['-q', 'sdist', '--format', 'gztar', '--dist-dir', bundle_dir_name],
)
# copy the generated targz to saved bundle directory and remove it from
# bentoml module directory
shutil.copytree(source_dir, target_path)
# clean up sdist build files
shutil.rmtree(source_dir)
def test_setup_requires_with_transitive_extra_dependency(
self, monkeypatch):
# Use case: installing a package with a build dependency on
# an already installed `dep[extra]`, which in turn depends
# on `extra_dep` (whose is not already installed).
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
# Create source distribution for `extra_dep`.
make_trivial_sdist(
os.path.join(temp_dir, 'extra_dep-1.0.tar.gz'),
'extra_dep', '1.0')
# Create source tree for `dep`.
dep_pkg = os.path.join(temp_dir, 'dep')
os.mkdir(dep_pkg)
path.build({
'setup.py':
DALS("""
import setuptools
setuptools.setup(
name='dep', version='2.0',
extras_require={'extra': ['extra_dep']},
)
"""),
'setup.cfg': '',
}, prefix=dep_pkg)
# "Install" dep.
run_setup(
os.path.join(dep_pkg, 'setup.py'), [str('dist_info')])
working_set.add_entry(dep_pkg)
# Create source tree for test package.
test_pkg = os.path.join(temp_dir, 'test_pkg')
test_setup_py = os.path.join(test_pkg, 'setup.py')
os.mkdir(test_pkg)
with open(test_setup_py, 'w') as fp:
fp.write(DALS(
'''
from setuptools import installer, setup
setup(setup_requires='dep[extra]')
'''))
# Check...
monkeypatch.setenv(str('PIP_FIND_LINKS'), str(temp_dir))
monkeypatch.setenv(str('PIP_NO_INDEX'), str('1'))
monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
run_setup(test_setup_py, [str('--version')])
def test_update_git_devstr(package_template, capsys):
"""Tests that the commit number in the package's version string updates
after git commits even without re-running setup.py.
"""
run_setup('setup.py', ['--version'])
stdout, stderr = capsys.readouterr()
version = stdout.strip()
m = _DEV_VERSION_RE.match(version)
assert m, ("Stdout did not match the version string pattern:"
"\n\n{0}\n\nStderr:\n\n{1}".format(stdout, stderr))
revcount = int(m.group(1))
import packagename
assert packagename.__version__ == version
# Make a silly git commit
with open('.test', 'w'):
pass
run_cmd('git', ['add', '.test'])
run_cmd('git', ['commit', '-m', 'test'])
import packagename.version
imp.reload(packagename.version)
# Previously this checked packagename.__version__, but in order for that to
# be updated we also have to re-import _astropy_init which could be tricky.
# Checking directly that the packagename.version module was updated is
# sufficient:
m = _DEV_VERSION_RE.match(packagename.version.version)
assert m
assert int(m.group(1)) == revcount + 1
# This doesn't test astropy_helpers.get_helpers.update_git_devstr directly
# since a copy of that function is made in packagename.version (so that it
# can work without astropy_helpers installed). In order to get test
# coverage on the actual astropy_helpers copy of that function just call it
# directly and compare to the value in packagename
from astropy_helpers.git_helpers import update_git_devstr
newversion = update_git_devstr(version, path=str(package_template))
assert newversion == packagename.version.version
def test_setup_requires_with_pep508_url(self, mock_index, monkeypatch):
monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
monkeypatch.setenv(str('PIP_INDEX_URL'), mock_index.url)
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
dep_sdist = os.path.join(temp_dir, 'dep.tar.gz')
make_trivial_sdist(dep_sdist, 'dependency', '42')
dep_url = path_to_url(dep_sdist, authority='localhost')
test_pkg = create_setup_requires_package(
temp_dir,
# Ignored (overriden by setup_attrs)
'python-xlib', '0.19',
setup_attrs=dict(
setup_requires='dependency @ %s' % dep_url))
test_setup_py = os.path.join(test_pkg, 'setup.py')
run_setup(test_setup_py, [str('--version')])
assert len(mock_index.requests) == 0
def test_bootstrap_from_archive(tmpdir, testpackage, capsys):
"""
Tests importing _astropy_helpers_test_ from a .tar.gz source archive
shipped alongside the package that uses it.
"""
orig_repo = tmpdir.mkdir('orig')
# Ensure ah_bootstrap is imported from the local directory
import ah_bootstrap
# Make a source distribution of the test package
with silence():
run_setup(str(testpackage.join('setup.py')),
['sdist', '--dist-dir=dist', '--formats=gztar'])
dist_dir = testpackage.join('dist')
for dist_file in dist_dir.visit('*.tar.gz'):
dist_file.copy(orig_repo)
with orig_repo.as_cwd():
# Write a test setup.py that uses ah_bootstrap; it also ensures that
# any previous reference to astropy_helpers is first wiped from
# sys.modules
args = 'path={0!r}'.format(os.path.basename(str(dist_file)))
orig_repo.join('setup.py').write(TEST_SETUP_PY.format(args=args))
run_setup('setup.py', [])
stdout, stderr = capsys.readouterr()
path = stdout.splitlines()[-1].strip()
# Installation from the .tar.gz should have resulted in a .egg
# directory that the _astropy_helpers_test_ package was imported from
eggs = glob.glob('*.egg')
assert eggs
egg = orig_repo.join(eggs[0])
assert os.path.isdir(str(egg))
a = os.path.normcase(path)
b = os.path.normcase(str(egg.join('_astropy_helpers_test_',
'__init__.py')))
assert a == b
def _submit_train_job(gcs_working_dir, version, params, region, scale_tier):
"""Module that submits a training job."""
# Run package setup
sandbox.run_setup('setup.py', ['-q', 'sdist'])
shutil.rmtree('trainer.egg-info') # Cleanup the directory not needed
# Copy package to GCS package path
package_path = '{}/packages/{}'.format(gcs_working_dir, PACKAGE_NAME)
_copy_to_gcs(os.path.join('dist', PACKAGE_NAME), package_path)
trainer_flags = [
'--gcs_working_dir', gcs_working_dir,
'--version', '{}'.format(version),
'--num_epochs', str(params['num_epochs']),
'--epochs_per_checkpoint', str(params['epochs_per_checkpoint']),
'--model_name', params['model_name']
]
training_inputs = {
'jobDir': gcs_working_dir,
'packageUris': package_path,
'pythonModule': 'cmle.trainer.task',
'args': trainer_flags,
'region': region
}
if scale_tier:
training_inputs['scale_tier'] = scale_tier
jobid = 'job_' + datetime.datetime.fromtimestamp(
time.time()).strftime('%Y%m%d_%H%M%S')
job_spec = {'jobId': jobid, 'trainingInput': training_inputs}
_, project_name = google.auth.default()
project_id = 'projects/{}'.format(project_name)
cloudml = discovery.build('ml', 'v1', cache_discovery=False)
request = cloudml.projects().jobs().create(body=job_spec,
parent=project_id)
try:
request.execute()
except errors.HttpError, err:
print('There was an error creating the training job.'
'Check the details:')
print(err._get_reason())
sys.exit(1)
def run_setup(self, *args, **kwargs):
"""In Python 3, on MacOS X, the import cache has to be invalidated
otherwise new extensions built with ``run_setup`` do not always get
picked up.
"""
try:
return sandbox.run_setup(*args, **kwargs)
finally:
import importlib
importlib.invalidate_caches()
def test_setup_requires(self):
"""Regression test for Distribute issue #318
Ensure that a package with setup_requires can be installed when
setuptools is installed in the user site-packages without causing a
SandboxViolation.
"""
test_pkg = create_setup_requires_package(os.getcwd())
test_setup_py = os.path.join(test_pkg, 'setup.py')
try:
with contexts.quiet():
with self.patched_setup_context():
run_setup(test_setup_py, ['install'])
except IndexError:
# Test fails in some cases due to bugs in Python
# See https://bitbucket.org/pypa/setuptools/issue/201
pass
def test_setup_requires(self):
"""Regression test for Distribute issue #318
Ensure that a package with setup_requires can be installed when
setuptools is installed in the user site-packages without causing a
SandboxViolation.
"""
test_pkg = create_setup_requires_package(os.getcwd())
test_setup_py = os.path.join(test_pkg, 'setup.py')
try:
with contexts.quiet():
with self.patched_setup_context():
run_setup(test_setup_py, ['install'])
except IndexError:
# Test fails in some cases due to bugs in Python
# See https://bitbucket.org/pypa/setuptools/issue/201
pass
def package_extension(cls, path_to_src, custom_display_name=None, keep_build_dir=False):
""" Function that creates The Extension.zip file from the give source path and returns
the path to the new Extension.zip
- path_to_src [String]: must include a setup.py, customize.py and config.py file.
- custom_display_name [String]: will give the Extension that display name. Default: name from setup.py file
- keep_build_dir [Boolean]: if True, dist/build/ will not be remove. Default: False
- The code will be packaged into a Built Distribution (.tar.gz) in the /dist directory
- The Extension.zip will also be produced in the /dist directory"""
# Ensure the src directory exists and we have WRITE access
cls.__validate_directory__(os.W_OK, path_to_src)
# Generate paths to files required to create extension
path_setup_py_file = os.path.join(path_to_src, BASE_NAME_SETUP_PY)
path_customize_py_file = os.path.join(path_to_src, os.path.basename(path_to_src), PATH_CUSTOMIZE_PY)
path_config_py_file = os.path.join(path_to_src, os.path.basename(path_to_src), PATH_CONFIG_PY)
path_output_dir = os.path.join(path_to_src, BASE_NAME_DIST_DIR)
path_extension_logo = os.path.join(path_to_src, PATH_ICON_EXTENSION_LOGO)
path_company_logo = os.path.join(path_to_src, PATH_ICON_COMPANY_LOGO)
LOG.info("Creating Built Distribution in /dist directory")
# Create the built distribution
use_setuptools.run_setup(setup_script=path_setup_py_file, args=["sdist", "--formats=gztar"])
LOG.info("Built Distribution (.tar.gz) created at: %s", path_output_dir)
# Create the extension
path_the_extension_zip = cls.create_extension(
path_setup_py_file=path_setup_py_file,
path_customize_py_file=path_customize_py_file,
path_config_py_file=path_config_py_file,
output_dir=path_output_dir,
custom_display_name=custom_display_name,
keep_build_dir=keep_build_dir,
path_extension_logo=path_extension_logo,
path_company_logo=path_company_logo
)
LOG.info("Extension created at: %s", path_the_extension_zip)
return path_the_extension_zip
def test_create_extension_display_name(fx_copy_fn_main_mock_integration):
path_fn_main_mock_integration = fx_copy_fn_main_mock_integration[1]
path_setup_py_file = os.path.join(path_fn_main_mock_integration,
package_helpers.BASE_NAME_SETUP_PY)
path_apiky_permissions_file = os.path.join(
path_fn_main_mock_integration,
package_helpers.BASE_NAME_APIKEY_PERMS_FILE)
output_dir = os.path.join(path_fn_main_mock_integration,
package_helpers.BASE_NAME_DIST_DIR)
use_setuptools.run_setup(setup_script=path_setup_py_file,
args=["sdist", "--formats=gztar"])
path_app_zip = package_helpers.create_extension(
path_setup_py_file, path_apiky_permissions_file, output_dir)
app_json = json.loads(sdk_helpers.read_zip_file(path_app_zip, "app.json"))
assert app_json.get("display_name") == "Main Mock Integration"
def run_setup(*args, **kwargs):
"""
In Python 3, on MacOS X, the import cache has to be invalidated otherwise
new extensions built with ``run_setup`` do not always get picked up.
"""
try:
return sandbox.run_setup(*args, **kwargs)
finally:
if sys.version_info[:2] >= (3, 3):
import importlib
importlib.invalidate_caches()
def test_setup_requires_honors_pip_env(self, mock_index, monkeypatch):
monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
monkeypatch.setenv(str('PIP_INDEX_URL'), mock_index.url)
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
test_pkg = create_setup_requires_package(
temp_dir, 'python-xlib', '0.19',
setup_attrs=dict(dependency_links=[]))
test_setup_cfg = os.path.join(test_pkg, 'setup.cfg')
with open(test_setup_cfg, 'w') as fp:
fp.write(DALS(
'''
[easy_install]
index_url = https://pypi.org/legacy/
'''))
test_setup_py = os.path.join(test_pkg, 'setup.py')
with pytest.raises(distutils.errors.DistutilsError):
run_setup(test_setup_py, [str('--version')])
assert len(mock_index.requests) == 1
assert mock_index.requests[0].path == '/python-xlib/'
def test_setup_requires_with_allow_hosts(self, mock_index):
''' The `allow-hosts` option in not supported anymore. '''
files = {
'test_pkg': {
'setup.py': DALS('''
from setuptools import setup
setup(setup_requires='python-xlib')
'''),
'setup.cfg': DALS('''
[easy_install]
allow_hosts = *
'''),
}
}
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
path.build(files, prefix=temp_dir)
setup_py = str(pathlib.Path(temp_dir, 'test_pkg', 'setup.py'))
with pytest.raises(distutils.errors.DistutilsError):
run_setup(setup_py, [str('--version')])
assert len(mock_index.requests) == 0
def test_bootstrap_from_archive(tmpdir, testpackage, capsys):
"""
Tests importing _astropy_helpers_test_ from a .tar.gz source archive
shipped alongside the package that uses it.
"""
orig_repo = tmpdir.mkdir('orig')
# Ensure ah_bootstrap is imported from the local directory
import ah_bootstrap
# Make a source distribution of the test package
with silence():
run_setup(str(testpackage.join('setup.py')),
['sdist', '--dist-dir=dist', '--formats=gztar'])
dist_dir = testpackage.join('dist')
for dist_file in dist_dir.visit('*.tar.gz'):
dist_file.copy(orig_repo)
with orig_repo.as_cwd():
# Write a test setup.py that uses ah_bootstrap; it also ensures that
# any previous reference to astropy_helpers is first wiped from
# sys.modules
args = 'path={0!r}'.format(os.path.basename(str(dist_file)))
orig_repo.join('setup.py').write(TEST_SETUP_PY.format(args=args))
run_setup('setup.py', [])
stdout, stderr = capsys.readouterr()
path = stdout.splitlines()[-1].strip()
# Installation from the .tar.gz should have resulted in a .egg
# directory that the _astropy_helpers_test_ package was imported from
eggs = glob.glob('*.egg')
assert eggs
egg = orig_repo.join(eggs[0])
assert os.path.isdir(str(egg))
assert path == str(egg.join('_astropy_helpers_test_', '__init__.py'))
def test_setup_requires(self):
"""Regression test for Distribute issue #318
Ensure that a package with setup_requires can be installed when
setuptools is installed in the user site-packages without causing a
SandboxViolation.
"""
test_setup_attrs = {
'name': 'test_pkg',
'version': '0.0',
'setup_requires': ['foobar'],
'dependency_links': [os.path.abspath(self.dir)]
}
test_pkg = os.path.join(self.dir, 'test_pkg')
test_setup_py = os.path.join(test_pkg, 'setup.py')
test_setup_cfg = os.path.join(test_pkg, 'setup.cfg')
os.mkdir(test_pkg)
f = open(test_setup_py, 'w')
f.write(
textwrap.dedent("""\
import setuptools
setuptools.setup(**%r)
""" % test_setup_attrs))
f.close()
foobar_path = os.path.join(self.dir, 'foobar-0.1.tar.gz')
make_trivial_sdist(
foobar_path,
textwrap.dedent("""\
import setuptools
setuptools.setup(
name='foobar',
version='0.1'
)
"""))
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdout = StringIO()
sys.stderr = StringIO()
try:
try:
reset_setup_stop_context(
lambda: run_setup(test_setup_py, ['install']))
except SandboxViolation:
self.fail('Installation caused SandboxViolation')
finally:
sys.stdout = old_stdout
sys.stderr = old_stderr
def create_package_from_setup_py(self, path_to_setup_py,
custom_package_name):
"""
Creates a package from Setup.py specified in the the given path
:param path_to_setup_py:
:return: Package Information: Contains package name, version and path
"""
if os.path.isfile(path_to_setup_py):
path_to_setup_py = os.path.abspath(path_to_setup_py)
setup_py_directory = os.path.dirname(path_to_setup_py)
elif os.path.isdir(path_to_setup_py):
setup_py_directory = os.path.abspath(path_to_setup_py)
path_to_setup_py = setup_py_directory + '/setup.py'
else:
raise Exception("Given path/file: " + path_to_setup_py +
" doesn't exist")
if not os.path.exists(path_to_setup_py):
raise Exception("setup.py doesn't exist in the given path: " +
path_to_setup_py)
sandbox.run_setup(path_to_setup_py, ['sdist'])
eggs = find_distributions(setup_py_directory)
version = None
for egg in eggs:
if custom_package_name in egg.egg_name():
version = egg.version
if not version:
raise Exception("{package_name} not found".format(
package_name=custom_package_name))
package_info = dict()
package_info[
'path'] = setup_py_directory + "/dist/" + custom_package_name + "-" + version + ".tar.gz"
package_info['version'] = version
package_info['name'] = custom_package_name
return package_info
def test_setup_requires_with_find_links_in_setup_cfg(
self, monkeypatch, use_legacy_installer,
with_dependency_links_in_setup_py):
monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
make_trivial_sdist(
os.path.join(temp_dir, 'python-xlib-42.tar.gz'),
'python-xlib', '42')
test_pkg = os.path.join(temp_dir, 'test_pkg')
test_setup_py = os.path.join(test_pkg, 'setup.py')
test_setup_cfg = os.path.join(test_pkg, 'setup.cfg')
os.mkdir(test_pkg)
with open(test_setup_py, 'w') as fp:
if with_dependency_links_in_setup_py:
dependency_links = [os.path.join(temp_dir, 'links')]
else:
dependency_links = []
fp.write(
DALS('''
from setuptools import installer, setup
if {use_legacy_installer}:
installer.fetch_build_egg = installer._legacy_fetch_build_egg
setup(setup_requires='python-xlib==42',
dependency_links={dependency_links!r})
''').format(
use_legacy_installer=use_legacy_installer, # noqa
dependency_links=dependency_links))
with open(test_setup_cfg, 'w') as fp:
fp.write(
DALS('''
[easy_install]
index_url = {index_url}
find_links = {find_links}
''').format(index_url=os.path.join(temp_dir, 'index'),
find_links=temp_dir))
run_setup(test_setup_py, [str('--version')])
def run(self):
"""
Remove previous builds, and build source and wheel distributions.
Also upload to pypi and push git tags.
"""
try:
self.status('Removing previous builds…')
rmtree(os.path.join(here, 'dist'))
except OSError:
pass
self.status('Building Source and Wheel (universal) distribution…')
sandbox.run_setup('setup.py', ['sdist', 'bdist_wheel', '--universal'])
self.status('Uploading the package to PyPI via Twine…')
twineupload("dist/*")
self.status('Pushing git tags…')
repo = Repo(os.getcwd())
tag_name = 'v{0}'.format(about['__version__'])
repo.create_tag(tag_name)
repo.remote('origin').push(tag_name)
sys.exit()
def _build_windows_distribution(self):
"""Create Windows source distribution
Contains executable installer and examples"""
import versioneer
version = versioneer.get_version()
self._remove_manifest()
run_setup("setup.py", ["bdist_wininst"])
self._remove_manifest()
shutil.copyfile("MANIFEST-WINDOWS.in", "MANIFEST.in")
run_setup("setup.py",
["sdist", "--dist-dir=dist-windows", "--formats=zip"])
shutil.move(
os.path.join("dist-windows", "FiPy-{}.zip".format(version)),
os.path.join("dist", "FiPy-{}.win32.zip".format(version)),
)
os.rmdir("dist-windows")
os.remove("MANIFEST.in")
def test_setup_requires_with_allow_hosts(self, mock_index):
''' The `allow-hosts` option in not supported anymore. '''
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
test_pkg = os.path.join(temp_dir, 'test_pkg')
test_setup_py = os.path.join(test_pkg, 'setup.py')
test_setup_cfg = os.path.join(test_pkg, 'setup.cfg')
os.mkdir(test_pkg)
with open(test_setup_py, 'w') as fp:
fp.write(DALS(
'''
from setuptools import setup
setup(setup_requires='python-xlib')
'''))
with open(test_setup_cfg, 'w') as fp:
fp.write(DALS(
'''
[easy_install]
allow_hosts = *
'''))
with pytest.raises(distutils.errors.DistutilsError):
run_setup(test_setup_py, [str('--version')])
assert len(mock_index.requests) == 0
def setup_and_run(temp_dir):
test_pkg = create_setup_requires_package(temp_dir)
test_setup_py = os.path.join(test_pkg, 'setup.py')
try:
stdout, stderr = quiet_context(lambda: reset_setup_stop_context(
# Don't even need to install the package, just running
# the setup.py at all is sufficient
lambda: run_setup(test_setup_py, ['--name'])))
except VersionConflict:
self.fail('Installing setup.py requirements caused '
'VersionConflict')
lines = stdout.splitlines()
self.assertGreater(len(lines), 0)
self.assert_(lines[-1].strip(), 'test_pkg')
def test_setup_requires(self):
"""Regression test for issue #318
Ensures that a package with setup_requires can be installed when
distribute is installed in the user site-packages without causing a
SandboxViolation.
"""
test_pkg = create_setup_requires_package(self.dir)
test_setup_py = os.path.join(test_pkg, 'setup.py')
try:
quiet_context(lambda: reset_setup_stop_context(lambda: run_setup(
test_setup_py, ['install'])))
except SandboxViolation:
self.fail('Installation caused SandboxViolation')
def setup_and_run(temp_dir):
test_pkg = create_setup_requires_package(temp_dir)
test_setup_py = os.path.join(test_pkg, 'setup.py')
try:
stdout, stderr = quiet_context(
lambda: reset_setup_stop_context(
# Don't even need to install the package, just running
# the setup.py at all is sufficient
lambda: run_setup(test_setup_py, ['--name'])
))
except VersionConflict:
self.fail('Installing setup.py requirements caused '
'VersionConflict')
lines = stdout.splitlines()
self.assertGreater(len(lines), 0)
self.assert_(lines[-1].strip(), 'test_pkg')
from modules.singleton import mrcnn, graph
from modules import utility
from models import result
from models.response_models import DetectionResult, DetectedBox, MatchScore, MatchResult
from setuptools import sandbox
from scipy.spatial.distance import cosine
import numpy as np
import os
import base64
try:
from . import ColorPy
except ImportError:
# setup ColorPy
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
sandbox.run_setup(SETUP_DIR + '/setup.py', ['build_ext', '--inplace'])
from . import ColorPy
def detect(images):
try:
# init variables
flag = False
TARGET_SIZE = (448, 448)
CONFIDENCE_THRESHOLD = 0.60
# init result list
match_result = list()
# load model & classes
model = mrcnn.get_model()
classes = mrcnn.get_classes()
def run(self):
from setuptools import sandbox
sandbox.run_setup('setup.py', ['bdist_wheel', '--python-tag', 'py3'])
def run(self, config, args):
packages = []
recipes = []
bundle_recipes = []
bundle_dirs = []
setup_args = ['sdist']
if not config.uninstalled:
m.error("Can only be run on cerbero-uninstalled")
store = PackagesStore(config)
cookbook = CookBook(config)
packages = list(args.bundlepackages)
for p in packages:
package = store.get_package(p)
if hasattr(package, 'list_packages'):
packages += package.list_packages()
packages = remove_list_duplicates(packages)
for p in packages:
package = store.get_package(p)
if hasattr(package, 'deps'):
packages += package.deps
packages = remove_list_duplicates(packages)
for p in packages:
package = store.get_package(p)
recipes += package.recipes_dependencies()
recipes += args.add_recipe
for r in recipes:
bundle_recipes += cookbook.list_recipe_deps(r)
bundle_recipes = remove_list_duplicates(bundle_recipes)
for p in packages:
setup_args.append('--package=' + p)
for r in bundle_recipes:
setup_args.append('--recipe=' + r.name)
if r.stype != SourceType.CUSTOM:
bundle_dirs.append(r.repo_dir)
if not args.no_bootstrap:
build_tools = BuildTools(config)
bs_recipes = build_tools.BUILD_TOOLS + \
build_tools.PLAT_BUILD_TOOLS.get(config.platform, [])
b_recipes = []
for r in bs_recipes:
b_recipes += cookbook.list_recipe_deps(r)
b_recipes = remove_list_duplicates(b_recipes)
for r in b_recipes:
if r.stype != SourceType.CUSTOM:
bundle_dirs.append(r.repo_dir)
setup_args.append('--source-dirs=' + ','.join(bundle_dirs))
command = str(config._relative_path('setup.py'))
run_setup(command, setup_args)
def test_bdist_egg_available_on_distutils_pkg(self, distutils_package):
run_setup('setup.py', ['bdist_egg'])
'regex',
'requests',
'torch',
]
# torch.hub doesn't build Cython components, so if they are not found then try
# to build them here
try:
import fairseq.data.token_block_utils_fast
except (ImportError, ModuleNotFoundError):
try:
import cython
import os
from setuptools import sandbox
sandbox.run_setup(
os.path.join(os.path.dirname(__file__), 'setup.py'),
['build_ext', '--inplace'],
)
except (ImportError, ModuleNotFoundError):
print(
'Unable to build Cython components. Please make sure Cython is '
'installed if the torch.hub model you are loading depends on it.')
for _model_type, _cls in MODEL_REGISTRY.items():
for model_name in _cls.hub_models().keys():
globals()[model_name] = functools.partial(
_cls.from_pretrained,
model_name,
)
# to simplify the interface we only expose named models
# globals()[_model_type] = _cls.from_pretrained
def test_setup_requires_override_nspkg(self, use_setup_cfg):
"""
Like ``test_setup_requires_overrides_version_conflict`` but where the
``setup_requires`` package is part of a namespace package that has
*already* been imported.
"""
with contexts.save_pkg_resources_state():
with contexts.tempdir() as temp_dir:
foobar_1_archive = os.path.join(temp_dir, 'foo.bar-0.1.tar.gz')
make_nspkg_sdist(foobar_1_archive, 'foo.bar', '0.1')
# Now actually go ahead an extract to the temp dir and add the
# extracted path to sys.path so foo.bar v0.1 is importable
foobar_1_dir = os.path.join(temp_dir, 'foo.bar-0.1')
os.mkdir(foobar_1_dir)
with tarfile.open(foobar_1_archive) as tf:
tf.extractall(foobar_1_dir)
sys.path.insert(1, foobar_1_dir)
dist = PRDistribution(foobar_1_dir,
project_name='foo.bar',
version='0.1')
working_set.add(dist)
template = DALS("""\
import foo # Even with foo imported first the
# setup_requires package should override
import setuptools
setuptools.setup(**%r)
if not (hasattr(foo, '__path__') and
len(foo.__path__) == 2):
print('FAIL')
if 'foo.bar-0.2' not in foo.__path__[0]:
print('FAIL')
""")
test_pkg = create_setup_requires_package(
temp_dir,
'foo.bar',
'0.2',
make_nspkg_sdist,
template,
use_setup_cfg=use_setup_cfg)
test_setup_py = os.path.join(test_pkg, 'setup.py')
with contexts.quiet() as (stdout, stderr):
try:
# Don't even need to install the package, just
# running the setup.py at all is sufficient
run_setup(test_setup_py, [str('--name')])
except pkg_resources.VersionConflict:
self.fail('Installing setup.py requirements '
'caused a VersionConflict')
assert 'FAIL' not in stdout.getvalue()
lines = stdout.readlines()
assert len(lines) > 0
assert lines[-1].strip() == 'test_pkg'
def test_upgrade(tmpdir, capsys):
# Run the testpackage fixture manually, since we use it multiple times in
# this test to make different versions of _astropy_helpers_test_
orig_dir = testpackage(tmpdir.mkdir('orig'))
# Make a test package that uses _astropy_helpers_test_
source = tmpdir.mkdir('source')
dist_dir = source.mkdir('dists')
orig_dir.copy(source.join('_astropy_helpers_test_'))
with source.as_cwd():
setup_py = TEST_SETUP_PY.format(args='auto_upgrade=True')
source.join('setup.py').write(setup_py)
# This will be used to later to fake downloading the upgrade package
source.join('setup.cfg').write(textwrap.dedent("""\
[easy_install]
find_links = {find_links}
""".format(find_links=str(dist_dir))))
# Make additional "upgrade" versions of the _astropy_helpers_test_
# package--one of them is version 0.2 and the other is version 0.1.1. The
# auto-upgrade should ignore version 0.2 but use version 0.1.1.
upgrade_dir_1 = testpackage(tmpdir.mkdir('upgrade_1'), version='0.2')
upgrade_dir_2 = testpackage(tmpdir.mkdir('upgrade_2'), version='0.1.1')
dists = []
# For each upgrade package go ahead and build a source distribution of it
# and copy that source distribution to a dist directory we'll use later to
# simulate a 'download'
for upgrade_dir in [upgrade_dir_1, upgrade_dir_2]:
with silence():
run_setup(str(upgrade_dir.join('setup.py')),
['sdist', '--dist-dir=dist', '--formats=gztar'])
dists.append(str(upgrade_dir.join('dist')))
for dist_file in upgrade_dir.visit('*.tar.gz'):
dist_file.copy(source.join('dists'))
# Monkey with the PackageIndex in ah_bootstrap so that it is initialized
# with the test upgrade packages, and so that it does not actually go out
# to the internet to look for anything
import ah_bootstrap
class FakePackageIndex(PackageIndex):
def __init__(self, *args, **kwargs):
PackageIndex.__init__(self, *args, **kwargs)
self.to_scan = dists
def find_packages(self, requirement):
# no-op
pass
ah_bootstrap.PackageIndex = FakePackageIndex
try:
with source.as_cwd():
# Now run the source setup.py; this test is similar to
# test_download_if_needed, but we explicitly check that the correct
# *version* of _astropy_helpers_test_ was used
run_setup('setup.py', [])
stdout, stderr = capsys.readouterr()
path = stdout.splitlines()[-1].strip()
eggs = glob.glob('*.egg')
assert eggs
egg = source.join(eggs[0])
assert os.path.isdir(str(egg))
assert path == str(egg.join('_astropy_helpers_test_',
'__init__.py'))
assert 'astropy_helpers_test-0.1.1-' in str(egg)
finally:
ah_bootstrap.PackageIndex = PackageIndex
def test_no_cython_buildext(tmpdir):
"""
Regression test for https://github.com/astropy/astropy-helpers/pull/35
This tests the custom build_ext command installed by astropy_helpers when
used with a project that has no Cython extensions (but does have one or
more normal C extensions).
"""
# In order for this test to test the correct code path we need to fool
# setup_helpers into thinking we don't have Cython installed
setup_helpers._module_state['have_cython'] = False
test_pkg = tmpdir.mkdir('test_pkg')
test_pkg.mkdir('_eva_').ensure('__init__.py')
# TODO: It might be later worth making this particular test package into a
# reusable fixture for other build_ext tests
# A minimal C extension for testing
test_pkg.join('_eva_').join('unit01.c').write(dedent("""\
#include <Python.h>
#ifndef PY3K
#if PY_MAJOR_VERSION >= 3
#define PY3K 1
#else
#define PY3K 0
#endif
#endif
#if PY3K
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"unit01",
NULL,
-1,
NULL
};
PyMODINIT_FUNC
PyInit_unit01(void) {
return PyModule_Create(&moduledef);
}
#else
PyMODINIT_FUNC
initunit01(void) {
Py_InitModule3("unit01", NULL, NULL);
}
#endif
"""))
test_pkg.join('setup.py').write(dedent("""\
from os.path import join
from setuptools import setup, Extension
from astropy_helpers.setup_helpers import register_commands
NAME = '_eva_'
VERSION = 0.1
RELEASE = True
cmdclassd = register_commands(NAME, VERSION, RELEASE)
setup(
name=NAME,
version=VERSION,
cmdclass=cmdclassd,
ext_modules=[Extension('_eva_.unit01',
[join('_eva_', 'unit01.c')])]
)
"""))
test_pkg.chdir()
run_setup('setup.py', ['build_ext', '--inplace'])
try:
import _eva_.unit01
dirname = os.path.abspath(os.path.dirname(_eva_.unit01.__file__))
assert dirname == str(test_pkg.join('_eva_'))
finally:
for modname in ['_eva_', '_eva_.unit01']:
try:
del sys.modules[modname]
except KeyError:
pass
)
from fairseq.models import MODEL_REGISTRY # noqa
# torch.hub doesn't build Cython components, so if they are not found then try
# to build them here
try:
import fairseq.data.token_block_utils_fast # noqa
except ImportError:
try:
import cython # noqa
import os
from setuptools import sandbox
sandbox.run_setup(
os.path.join(os.path.dirname(__file__), "setup.py"),
["build_ext", "--inplace"],
)
except ImportError:
print(
"Unable to build Cython components. Please make sure Cython is "
"installed if the torch.hub model you are loading depends on it."
)
# automatically expose models defined in FairseqModel::hub_models
for _model_type, _cls in MODEL_REGISTRY.items():
for model_name in _cls.hub_models().keys():
globals()[model_name] = functools.partial(
_cls.from_pretrained,
model_name,
)
args = parser.parse_args()
print(args)
if not isdir(args.dirname):
print('Please specify existing directory with pipelines! ', args.dirname,
' directory does not exist.')
print(
'Usage: ./run_pipeline.py directory_with_pipelines --pipeline-name my_pipeline'
)
print('--pipeline-name parameter is optional')
sys.exit(-100)
if args.pipeline_name:
if not isdir(args.dirname + '/' + args.pipeline_name):
print('Please specify existing pipeline name as --pipeline-name ',
args.dirname + '/' + args.pipeline_name,
' drectory does not exist.')
print(
'Usage: ./run_pipeline.py directory_with_pipelines --pipeline-name my_pipeline'
)
print('--pipeline-name parameter is optional')
sys.exit(-100)
import mlflow
mlflow.set_tracking_uri("databricks")
from setuptools import sandbox
sandbox.run_setup('setup.py', ['clean', 'bdist_wheel'])
from databrickslabs_mlflowdepl import dev_cicd_pipeline
dev_cicd_pipeline.main(args.dirname, args.pipeline_name)
else:
datapath = os.environ.get('TFIO_DATAPATH')
if datapath is not None:
for rootname, _, filenames in os.walk(
os.path.join(datapath, "tensorflow_io")):
if (not fnmatch.fnmatch(rootname, "*test*")
and not fnmatch.fnmatch(rootname, "*runfiles*")):
for filename in fnmatch.filter(filenames, "*.so"):
src = os.path.join(rootname, filename)
dst = os.path.join(
rootpath,
os.path.relpath(os.path.join(rootname, filename),
datapath))
print("setup.py - copy {} to {}".format(src, dst))
shutil.copyfile(src, dst)
print("setup.py - run sandbox.run_setup {} {}".format(
os.path.join(rootpath, "setup.py"), sys.argv[1:]))
sandbox.run_setup(os.path.join(rootpath, "setup.py"), sys.argv[1:])
if not os.path.exists("dist"):
os.makedirs("dist")
for f in os.listdir(os.path.join(rootpath, "dist")):
print("setup.py - copy {} to {}".format(os.path.join(rootpath, "dist", f),
os.path.join("dist", f)))
shutil.copyfile(os.path.join(rootpath, "dist", f), os.path.join("dist", f))
print("setup.py - remove {}".format(rootpath))
shutil.rmtree(rootpath)
print("setup.py - complete")
with open(init_file, "w") as outfile:
outfile.write("".join(package_init))
reload(pyrsched.server)
setup_version, git_version, pypi_version = get_versions()
if pypi_version < setup_version:
print("The package on PyPI needs to be updated.")
r = input("Build Package and update on PyPI? [Yn] ") or "Y"
if r.upper() == "Y":
# clear dist directory
dist_path = Path("dist").resolve()
for f in dist_path.iterdir():
f.unlink()
# build package
from setuptools.sandbox import run_setup
run_setup("setup.py", ["sdist", "bdist_wheel"])
# upload package
from twine import cli
from twine import exceptions
try:
twine_res = cli.dispatch(["upload", "dist/*"])
except (exceptions.TwineException, requests.HTTPError) as exc:
print("{}: {}".format(exc.__class__.__name__, exc.args[0]))
print(f"Twine response: {twine_res}")
setup_version, git_version, pypi_version = get_versions()
print("all done...")
# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright
# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''
# A unique identification for the text.
#
# epub_uid = ''
# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']
# custom
extensions += ['sphinx.ext.todo']
todo_include_todos = True
extensions += ['sphinx.ext.autosummary']
extensions += ['sphinx.ext.imgmath']
numpydoc_show_class_members = False
html_theme = "sphinx_rtd_theme"
import sys, os
sys.path.insert(0, os.path.abspath('../../'))
from setuptools import sandbox
sandbox.run_setup(os.path.abspath('../../setup.py'), ['build_ext', '-i'])
autoclass_content = 'both'
def test_bdist_egg_available_on_distutils_pkg(self, distutils_package):
run_setup('setup.py', ['bdist_egg'])
def mk_api_pkg(
openapi_spec: dict = None,
openapi_url: str = None,
pkg_name: str = None,
pkg_version: str = None,
pkg_description: str = None,
pkg_author: str = None,
pkg_license: str = None,
):
"""
Build a pip installable package containing the functions to consume a running
webservice application executed with py2http.
:param openapi_spec: The openapi specification for the webservice.
:type openapi_spec: dict, optional
:param openapi_url: The url to fetch the openapi specification for the webservice.
:type openapi_url: str, optional
:param pkg_name: The name of the package. If no name is provided, default to
'apipkg{timestamp}'
:type pkg_name: str, optional
:param pkg_version: The version number of the package. If no version is provided,
default to 1.0.0
:type pkg_version: str, optional
:param pkg_description: The description of the package.
:type pkg_description: str, optional
:param pkg_author: The author of the package. If no author provided, default to
'API Package Maker'
:type pkg_author: str, optional
:param pkg_license: The license for the package. If no license provided, default
to 'Apache'
:type pkg_license: str, optional
"""
def mk_str_element(value: str):
return value if value is None else f"'{value}'"
def mk_pkg_name():
dt = datetime.now(timezone.utc)
utc_time = dt.replace(tzinfo=timezone.utc)
utc_timestamp = int(utc_time.timestamp())
return f'apipkg{utc_timestamp}'
def create_file(filepath, content):
with open(filepath, 'w') as file:
print(content, file=file)
tempdir = tempfile.mkdtemp()
try:
api = HttpClient(openapi_spec=openapi_spec, url=openapi_url)
paths_spec = api.openapi_spec.get('paths')
if not paths_spec:
raise RuntimeError('The API is empty')
func_names = [path[1:] for path in paths_spec]
funcs_code = [
FUNC_TPL.format(func_name=func_name) for func_name in func_names
]
funcs_file_code = FUNCS_FILE_TPL.format(
openapi_spec=mk_str_element(openapi_spec),
openapi_url=mk_str_element(openapi_url),
funcs=''.join(funcs_code),
)
init_file_code = INIT_FILE_TPL.format(funcs=', '.join(func_names))
pkg_name = pkg_name or mk_pkg_name()
module_dir = os.path.join(tempdir, pkg_name)
os.makedirs(module_dir)
create_file(filepath=os.path.join(module_dir, f'__init__.py'),
content=init_file_code)
create_file(filepath=os.path.join(module_dir, f'funcs.py'),
content=funcs_file_code)
server_url = api.openapi_spec['servers'][0]['url']
pkg_version = pkg_version or '1.0.0'
setup_cfg_content = SETUP_CFG_TPL.format(
name=pkg_name,
version=pkg_version,
description=pkg_description or
f'A client API to consume the webservices exposed at {server_url}',
author=pkg_author or 'API Package Maker',
license=pkg_license or 'Apache',
)
create_file(filepath=os.path.join(tempdir, f'setup.cfg'),
content=setup_cfg_content)
create_file(filepath=os.path.join(tempdir, f'setup.py'),
content=SETUP_PY)
os.chdir(tempdir)
sandbox.run_setup('setup.py', ['sdist'])
pkg_filename = f'{pkg_name}-{pkg_version}.tar.gz'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
shutil.copy(src=os.path.join(tempdir, 'dist', pkg_filename),
dst=OUTPUT_DIR)
return os.path.join(OUTPUT_DIR, pkg_filename)
finally:
shutil.rmtree(tempdir)
