def pep8_report(self):
"""
Outputs PEP8 report to screen and pep8.txt.
"""
verbose = '--quiet' not in sys.argv
if verbose:
# Hook into stdout.
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
# Run Pep8 checks.
pep8.options, pep8.args = pep8.process_options()
pep8.args = self.packages
pep8.options.repeat = True
for package in self.packages:
pep8.input_dir(package)
# Restore stdout.
sys.stdout = old_stdout
# Save result to pep8.txt.
result = mystdout.getvalue()
output = open('pep8.txt', 'w')
output.write(result)
output.close()
# Return Pep8 result
if result:
log.info("\nPEP8 Report:")
log.info(result)
def dist2_to_args(config, dist=None):
'''
Converts from distutil2 to distutil1 options.
'''
dest = {}
for (section, option), func in D2TO1:
if func is None:
log.warn('key %r not yet implemented', (section, option))
continue
if not func:
continue
try:
value = config[section][option]
except KeyError:
log.info('%r not found', (section, option))
continue
if func == 'let':
if (section, option) == ('global', 'commands'):
assign_cmds(config, dest, value, dist)
continue
if (section, option) == ('global', 'compilers'):
continue
raise Exception('Not implemented')
func(config, dest, value)
return dest
def run_tests(self):
import sys, os
if sys.version_info[0] == 3:
rootdir = os.path.dirname(os.path.abspath(__file__))
if rootdir in sys.path:
sys.path.remove(rootdir)
ei_cmd = self.get_finalized_command('egg_info')
egg_name = ei_cmd.egg_name.replace('-', '_')
to_remove = []
for dirname in sys.path:
bn = os.path.basename(dirname)
if bn.startswith(egg_name + "-"):
to_remove.append(dirname)
for dirname in to_remove:
log.info("removing installed %r from sys.path before testing"%(dirname,))
sys.path.remove(dirname)
from PyObjCTest.loader import makeTestSuite
import unittest
#import pprint; pprint.pprint(sys.path)
unittest.main(None, None, [unittest.__file__]+self.test_args)
def rename_python_scripts(self):
"Rename main executable python script without .py extension"
for script in self.get_outputs():
if script.endswith(".py"):
info('renaming the python script %s -> %s' % (
script, script[:-3]))
shutil.move(script, script[:-3])
def _get_package_version(self):
"""
Attempt to get the most correct current version of Sentry.
"""
pkg_path = os.path.join(self.work_path, 'src')
sys.path.insert(0, pkg_path)
try:
import sentry
except Exception:
version = None
build = None
else:
log.info("pulled version information from 'sentry' module".format(
sentry.__file__))
version = VERSION
build = sentry.__build__
finally:
sys.path.pop(0)
if not (version and build):
try:
with open(self.sentry_package_json_path) as fp:
data = json.loads(fp.read())
except Exception:
pass
else:
log.info("pulled version information from 'sentry-package.json'")
version, build = data['version'], data['build']
return {
'version': version,
'build': build,
}
def run(self):
from distutils import log
from distutils.filelist import FileList
global CLEANUP
distutils.command.clean.clean.run(self)
if self.all:
fl = FileList()
fl.include_pattern("*.pyc", 0)
fl.include_pattern("*.pyd", 0)
fl.include_pattern("*.so", 0)
CLEANUP += fl.files
for f in CLEANUP:
if os.path.isdir(f):
try:
if not self.dry_run and os.path.exists(f):
os.rmdir(f)
log.info("removing '%s'", f)
except IOError:
log.warning("unable to remove '%s'", f)
else:
try:
if not self.dry_run and os.path.exists(f):
os.remove(f)
log.info("removing '%s'", f)
except IOError:
log.warning("unable to remove '%s'", f)
def _get_increment_kwargs(git_dir, tag):
"""Calculate the sort of semver increment needed from git history.
Every commit from HEAD to tag is consider for Sem-Ver metadata lines.
See the pbr docs for their syntax.
:return: a dict of kwargs for passing into SemanticVersion.increment.
"""
result = {}
if tag:
version_spec = tag + "..HEAD"
else:
version_spec = "HEAD"
changelog = git._run_git_command(['log', version_spec], git_dir)
header_len = len(' sem-ver:')
commands = [line[header_len:].strip() for line in changelog.split('\n')
if line.lower().startswith(' sem-ver:')]
symbols = set()
for command in commands:
symbols.update([symbol.strip() for symbol in command.split(',')])
def _handle_symbol(symbol, symbols, impact):
if symbol in symbols:
result[impact] = True
symbols.discard(symbol)
_handle_symbol('bugfix', symbols, 'patch')
_handle_symbol('feature', symbols, 'minor')
_handle_symbol('deprecation', symbols, 'minor')
_handle_symbol('api-break', symbols, 'major')
for symbol in symbols:
log.info('[pbr] Unknown Sem-Ver symbol %r' % symbol)
# We don't want patch in the kwargs since it is not a keyword argument -
# its the default minimum increment.
result.pop('patch', None)
return result
def run(self):
version = get_version()
if version is None:
sys.exit(1)
weekly_version = 'weekly-v%s' % version
log.info('tagging as %s', weekly_version)
subprocess.check_call('git tag -f'.split() + [weekly_version])
def get_snappy(self):
def build_snappy():
cppflags = '-fPIC'
env = os.environ.copy()
env['CPPFLAGS'] = ' '.join(x for x in (cppflags, env.get('CPPFLAGS', None)) if x)
log.info('Building snappy...')
exec_process(['sh', 'autogen.sh'], cwd=self.snappy_dir, env=env, silent=not self.snappy_verbose_build)
exec_process(['./configure'], cwd=self.snappy_dir, env=env, silent=not self.snappy_verbose_build)
exec_process(['make'], cwd=self.snappy_dir, env=env, silent=not self.snappy_verbose_build)
if self.snappy_force_fetch:
rmtree('deps')
if not os.path.exists(self.snappy_dir):
try:
download_snappy(self.snappy_dir, self.snappy_version, self.snappy_url)
except BaseException:
rmtree('deps')
raise
build_snappy()
else:
if self.snappy_clean_compile:
exec_process(['make', 'clean'], cwd=self.snappy_dir)
exec_process(['make', 'distclean'], cwd=self.snappy_dir)
if not os.path.exists(self.snappy_lib):
log.info('snappy needs to be compiled.')
build_snappy()
else:
log.info('No need to build snappy.')
def run_cython(source, depends=(), includes=(),
destdir_c=None, destdir_h=None,
wdir=None, force=False, VERSION=None):
from glob import glob
from distutils import log
from distutils import dep_util
from distutils.errors import DistutilsError
target = os.path.splitext(source)[0]+'.c'
cwd = os.getcwd()
try:
if wdir: os.chdir(wdir)
alldeps = [source]
for dep in depends:
alldeps += glob(dep)
if not (force or dep_util.newer_group(alldeps, target)):
log.debug("skipping '%s' -> '%s' (up-to-date)",
source, target)
return
finally:
os.chdir(cwd)
if not chk_cython(VERSION):
raise DistutilsError("requires Cython>=%s" % VERSION)
log.info("cythonizing '%s' -> '%s'", source, target)
from cythonize import cythonize
err = cythonize(source,
includes=includes,
destdir_c=destdir_c,
destdir_h=destdir_h,
wdir=wdir)
if err:
raise DistutilsError(
"Cython failure: '%s' -> '%s'" % (source, target))
def run(self):
_clean.run(self)
import fnmatch
# kill temporary files
patterns = [
# generic tempfiles
'*~', '*.bak', '*.pyc',
# tempfiles generated by ANTLR runs
't[0-9]*Lexer.py', 't[0-9]*Parser.py',
'*.tokens', '*__.g',
]
for path in ('antlr3', 'unittests', 'tests'):
path = os.path.join(os.path.dirname(__file__), path)
if os.path.isdir(path):
for root, dirs, files in os.walk(path, topdown=True):
graveyard = []
for pat in patterns:
graveyard.extend(fnmatch.filter(files, pat))
for name in graveyard:
filePath = os.path.join(root, name)
try:
log.info("removing '%s'", filePath)
os.unlink(filePath)
except OSError, exc:
log.warn(
"Failed to delete '%s': %s",
filePath, exc
)
def remove_tree(directory, verbose=1, dry_run=0):
"""Recursively remove an entire directory tree.
Any errors are ignored (apart from being reported to stdout if 'verbose'
is true).
"""
from distutils.util import grok_environment_error
global _path_created
if verbose >= 1:
log.info("removing '%s' (and everything under it)", directory)
if dry_run:
return
cmdtuples = []
_build_cmdtuple(directory, cmdtuples)
for cmd in cmdtuples:
try:
cmd[0](cmd[1])
# remove dir from cache if it's already there
abspath = os.path.abspath(cmd[1])
if abspath in _path_created:
del _path_created[abspath]
except (IOError, OSError) as exc:
log.warn(grok_environment_error(
exc, "error removing %s: " % directory))
def install_namespaces(self):
nsp = self._get_all_ns_packages()
if not nsp: return
filename,ext = os.path.splitext(self.target)
filename += '-nspkg.pth'; self.outputs.append(filename)
log.info("Installing %s",filename)
if not self.dry_run:
f = open(filename,'wt')
for pkg in nsp:
# ensure pkg is not a unicode string under Python 2.7
pkg = str(pkg)
pth = tuple(pkg.split('.'))
trailer = '\n'
if '.' in pkg:
trailer = (
"; m and setattr(sys.modules[%r], %r, m)\n"
% ('.'.join(pth[:-1]), pth[-1])
)
f.write(
"import sys,types,os; "
"p = os.path.join(sys._getframe(1).f_locals['sitedir'], "
"*%(pth)r); "
"ie = os.path.exists(os.path.join(p,'__init__.py')); "
"m = not ie and "
"sys.modules.setdefault(%(pkg)r,types.ModuleType(%(pkg)r)); "
"mp = (m or []) and m.__dict__.setdefault('__path__',[]); "
"(p not in mp) and mp.append(p)%(trailer)s"
% locals()
)
f.close()
def format_changelog(self, version):
""" Write new header at beginning of changelog file with the specified
'version' and the current date.
Return the changelog content for the current release.
"""
from datetime import datetime
log.info("formatting changelog")
changes = []
with io.open(self.changelog_name, "r+", encoding="utf-8") as f:
for ln in f:
if ln.startswith(self.changelog_header):
break
else:
changes.append(ln)
if not self.dry_run:
f.seek(0)
content = f.read()
f.seek(0)
f.write(u"%s%s\n\n%s\n\n%s" % (
self.changelog_header,
version,
datetime.today().strftime(self.changelog_date_fmt),
content))
return u"".join(changes)
def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
mode='w'):
"""Create a zip file from all the files under 'base_dir'. The output
zip file will be named 'base_dir' + ".zip". Uses either the "zipfile"
Python module (if available) or the InfoZIP "zip" utility (if installed
and found on the default search path). If neither tool is available,
raises DistutilsExecError. Returns the name of the output zip file.
"""
import zipfile
mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
def visit(z, dirname, names):
for name in names:
path = os.path.normpath(os.path.join(dirname, name))
if os.path.isfile(path):
p = path[len(base_dir) + 1:]
if not dry_run:
z.write(path, p)
log.debug("adding '%s'" % p)
compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
if not dry_run:
z = zipfile.ZipFile(zip_filename, mode, compression=compression)
for dirname, dirs, files in os.walk(base_dir):
visit(z, dirname, files)
z.close()
else:
for dirname, dirs, files in os.walk(base_dir):
visit(None, dirname, files)
return zip_filename
def run(self):
if self.download_bootstrap_script is True:
# Let's update the bootstrap-script to the version defined to be
# distributed. See BOOTSTRAP_SCRIPT_DISTRIBUTED_VERSION above.
url = (
'https://github.com/saltstack/salt-bootstrap/raw/{0}'
'/bootstrap-salt.sh'.format(
BOOTSTRAP_SCRIPT_DISTRIBUTED_VERSION
)
)
deploy_path = os.path.join(
SETUP_DIRNAME,
'salt',
'cloud',
'deploy',
'bootstrap-salt.sh'
)
log.info(
'Updating bootstrap-salt.sh.'
'\n\tSource: {0}'
'\n\tDestination: {1}'.format(
url,
deploy_path
)
)
try:
import requests
req = requests.get(url)
if req.status_code == 200:
script_contents = req.text.encode(req.encoding)
else:
log.error(
'Failed to update the bootstrap-salt.sh script. HTTP '
'Error code: {0}'.format(
req.status_code
)
)
except ImportError:
req = urlopen(url)
if req.getcode() == 200:
script_contents = req.read()
else:
log.error(
'Failed to update the bootstrap-salt.sh script. HTTP '
'Error code: {0}'.format(
req.getcode()
)
)
try:
with open(deploy_path, 'w') as fp_:
fp_.write(script_contents)
except (OSError, IOError) as err:
log.error(
'Failed to write the updated script: {0}'.format(err)
)
# Let's the rest of the build command
Sdist.run(self)
def format_changelog(self, version):
""" Write new header at beginning of changelog file with the specified
'version' and the current date.
Return the changelog content for the current release.
"""
from datetime import datetime
log.info("formatting changelog")
changes = []
with io.open(self.changelog_name, "r+", encoding="utf-8") as f:
for ln in f:
if self.version_RE.match(ln):
break
else:
changes.append(ln)
if not self.dry_run:
f.seek(0)
content = f.read()
date = datetime.today().strftime(self.date_fmt)
f.seek(0)
header = self.header_fmt % (version, date)
f.write(header + u"\n" + u"-"*len(header) + u"\n\n" + content)
return u"".join(changes)
def rm(f):
try:
os.unlink(f)
log.info("removing " + f)
except:
log.warn("can't clean " + f),
RegError = winreg.error
except ImportError:
try:
import win32api
import win32con
_can_read_reg = True
hkey_mod = win32con
RegOpenKeyEx = win32api.RegOpenKeyEx
RegEnumKey = win32api.RegEnumKey
RegEnumValue = win32api.RegEnumValue
RegError = win32api.error
except ImportError:
log.info("Warning: Can't read registry to find the "
"necessary compiler setting\n"
"Make sure that Python modules winreg, "
"win32api or win32con are installed.")
pass
if _can_read_reg:
HKEYS = (hkey_mod.HKEY_USERS,
hkey_mod.HKEY_CURRENT_USER,
hkey_mod.HKEY_LOCAL_MACHINE,
hkey_mod.HKEY_CLASSES_ROOT)
def read_keys(base, key):
"""Return list of registry keys."""
try:
handle = RegOpenKeyEx(base, key)
except RegError:
return None
def run(self):
if not os.path.exists('../pyobjc-core'):
raise DistutilsError(
"Run me in a complete checkout of the pyobjc trunk")
frameworkList = [dn for dn in os.listdir('..') if dn.startswith('pyobjc-framework') ]
#frameworkList = ['pyobjc-framework-Cocoa']
root_menu=[
('Home', '/index.html'),
('News', '/news.html'),
('About', '/about.html'),
('History&People', '/people.html'),
('Links', '/links.html'),
]
generator = sitegen.SiteGenerator('templates', 'htdocs')
if os.path.exists('htdocs'):
shutil.rmtree('htdocs')
os.mkdir('htdocs')
log.info("Copying static resources")
for subdir in os.listdir('resources'):
if subdir.startswith('.'): continue
log.info(" - %s" % (subdir,))
generator.copy(os.path.join('resources', subdir), subdir)
log.info("Copying static HTML")
for fn in os.listdir('static'):
if fn.startswith('.'): continue
log.info(" - %s" % (fn,))
generator.copyReST(os.path.join('static', fn), os.path.splitext(fn)[0] + '.html', bottommenu=root_menu)
log.info("Processing news items")
newsItems = news.parseNews('news')
log.info("Emitting news")
generator.emitHTML("/news.html", "news.html", news=newsItems, bottommenu=root_menu)
for item in newsItems:
# Generate seperate files for the news items, will be used from an RSS feed.
generator.emitHTML('/news/%s.html'%(item['basename'],), 'news-item.html', newsitem=item, bottommenu=root_menu)
log.info("Emitting homepage")
generator.emitHTML("/index.html", "site-index.html",
pyobjc_version='2.2b2',
pyobjc_release_date='May 26th 2009',
news=news.newsSelection(newsItems),
bottommenu=root_menu)
docs.generateDocs(generator, '/documentation', '..', ['pyobjc-core'] + frameworkList)
samples.generateSamples(generator, '/examples', '..', frameworkList)
def run(self):
"""
Run the tests!
"""
# Install the runtime dependencies.
if self.distribution.install_requires:
self.distribution.fetch_build_eggs(
self.distribution.install_requires)
# Ensure there is a doc path
if self.docs_path is None:
cfg_docs_dir = self.distribution.get_option_dict('build_docs').get(
'source_dir', None)
# Some affiliated packages use this.
# See astropy/package-template#157
if cfg_docs_dir is not None and os.path.exists(cfg_docs_dir[1]):
self.docs_path = os.path.abspath(cfg_docs_dir[1])
# fall back on a default path of "docs"
elif os.path.exists('docs'): # pragma: no cover
self.docs_path = os.path.abspath('docs')
# Build a testing install of the package
self._build_temp_install()
# Install the test dependencies
# NOTE: we do this here after _build_temp_install because there is
# a weird but which occurs if psutil is installed in this way before
# astropy is built, Cython can have segmentation fault. Strange, eh?
if self.distribution.tests_require:
self.distribution.fetch_build_eggs(self.distribution.tests_require)
# Copy any additional dependencies that may have been installed via
# tests_requires or install_requires. We then pass the
# add_local_eggs_to_path=True option to package.test() to make sure the
# eggs get included in the path.
if os.path.exists('.eggs'):
shutil.copytree('.eggs', os.path.join(self.testing_path, '.eggs'))
# This option exists so that we can make sure that the tests don't
# write to an installed location.
if self.readonly:
log.info(
'changing permissions of temporary installation to read-only')
self._change_permissions_testing_path(writable=False)
# Run everything in a try: finally: so that the tmp dir gets deleted.
try:
# Construct this modules testing command
cmd = self.generate_testing_command()
# Run the tests in a subprocess--this is necessary since
# new extension modules may have appeared, and this is the
# easiest way to set up a new environment
testproc = subprocess.Popen([sys.executable, '-c', cmd],
cwd=self.testing_path,
close_fds=False)
retcode = testproc.wait()
except KeyboardInterrupt:
import signal
# If a keyboard interrupt is handled, pass it to the test
# subprocess to prompt pytest to initiate its teardown
testproc.send_signal(signal.SIGINT)
retcode = testproc.wait()
finally:
# Remove temporary directory
if self.readonly:
self._change_permissions_testing_path(writable=True)
shutil.rmtree(self.tmp_dir)
raise SystemExit(retcode)
def build_extension(self, ext):
sources = ext.sources
if sources is None or not isinstance(sources, (list, tuple)):
raise DistutilsSetupError(
"in 'ext_modules' option (extension '%s'), "
"'sources' must be present and must be "
"a list of source filenames" % ext.name)
sources = list(sources)
ext_path = self.get_ext_fullpath(ext.name)
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_path, 'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
# First, scan the sources for SWIG definition files (.i), run
# SWIG on 'em to create .c files, and modify the sources list
# accordingly.
sources = self.swig_sources(sources, ext)
# Next, compile the source code to object files.
# XXX not honouring 'define_macros' or 'undef_macros' -- the
# CCompiler API needs to change to accommodate this, and I
# want to do one thing at a time!
# Two possible sources for extra compiler arguments:
# - 'extra_compile_args' in Extension object
# - CFLAGS environment variable (not particularly
# elegant, but people seem to expect it and I
# guess it's useful)
# The environment variable should take precedence, and
# any sensible compiler will give precedence to later
# command line args. Hence we combine them in order:
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef,))
objects = self.compiler.compile(sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=ext.include_dirs,
debug=self.debug,
extra_postargs=extra_args,
depends=ext.depends)
# XXX -- this is a Vile HACK!
#
# The setup.py script for Python on Unix needs to be able to
# get this list so it can perform all the clean up needed to
# avoid keeping object files around when cleaning out a failed
# build of an extension module. Since Distutils does not
# track dependencies, we have to get rid of intermediates to
# ensure all the intermediates will be properly re-built.
#
self._built_objects = objects[:]
# Now link the object files together into a "shared object" --
# of course, first we have to figure out all the other things
# that go into the mix.
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
# Detect target language, if not provided
language = ext.language or self.compiler.detect_language(sources)
self.compiler.link_shared_object(
objects, ext_path,
libraries=self.get_libraries(ext),
library_dirs=ext.library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=language)
def run(self):
if cmake_build.built:
return
cmake_build.built = True
if not os.path.exists(CMAKE_BUILD_DIR):
os.makedirs(CMAKE_BUILD_DIR)
with cd(CMAKE_BUILD_DIR):
build_type = 'Release'
# configure
cmake_args = [
CMAKE,
'-DPYTHON_INCLUDE_DIR={}'.format(sysconfig.get_python_inc()),
'-DPYTHON_EXECUTABLE={}'.format(sys.executable),
'-DBUILD_ONNX_PYTHON=ON',
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DONNX_NAMESPACE={}'.format(ONNX_NAMESPACE),
'-DPY_EXT_SUFFIX={}'.format(
sysconfig.get_config_var('EXT_SUFFIX') or ''),
]
if COVERAGE:
cmake_args.append('-DONNX_COVERAGE=ON')
if COVERAGE or DEBUG:
# in order to get accurate coverage information, the
# build needs to turn off optimizations
build_type = 'Debug'
cmake_args.append('-DCMAKE_BUILD_TYPE=%s' % build_type)
if WINDOWS:
cmake_args.extend([
# we need to link with libpython on windows, so
# passing python version to window in order to
# find python in cmake
'-DPY_VERSION={}'.format(
'{0}.{1}'.format(*sys.version_info[:2])),
])
if USE_MSVC_STATIC_RUNTIME:
cmake_args.append('-DONNX_USE_MSVC_STATIC_RUNTIME=ON')
if platform.architecture()[0] == '64bit':
cmake_args.extend(['-A', 'x64', '-T', 'host=x64'])
else:
cmake_args.extend(['-A', 'Win32', '-T', 'host=x86'])
if ONNX_ML:
cmake_args.append('-DONNX_ML=1')
if ONNX_VERIFY_PROTO3:
cmake_args.append('-DONNX_VERIFY_PROTO3=1')
if ONNX_BUILD_TESTS:
cmake_args.append('-DONNX_BUILD_TESTS=ON')
if 'CMAKE_ARGS' in os.environ:
extra_cmake_args = shlex.split(os.environ['CMAKE_ARGS'])
# prevent crossfire with downstream scripts
del os.environ['CMAKE_ARGS']
log.info('Extra cmake args: {}'.format(extra_cmake_args))
cmake_args.extend(extra_cmake_args)
cmake_args.append(TOP_DIR)
subprocess.check_call(cmake_args)
build_args = [CMAKE, '--build', os.curdir]
if WINDOWS:
build_args.extend(['--config', build_type])
build_args.extend(['--', '/maxcpucount:{}'.format(self.jobs)])
else:
build_args.extend(['--', '-j', str(self.jobs)])
subprocess.check_call(build_args)
def run(self):
if cmake_build.built:
return
cmake_build.built = True
if not os.path.exists(CMAKE_BUILD_DIR):
os.makedirs(CMAKE_BUILD_DIR)
with cd(CMAKE_BUILD_DIR):
build_type = "Release"
# configure
cmake_args = [
CMAKE,
f"-DPYTHON_INCLUDE_DIR={sysconfig.get_python_inc()}",
f"-DPYTHON_EXECUTABLE={sys.executable}",
"-DBUILD_ONNX_PYTHON=ON",
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
f"-DONNX_NAMESPACE={ONNX_NAMESPACE}",
f"-DPY_EXT_SUFFIX={sysconfig.get_config_var('EXT_SUFFIX') or ''}",
]
if COVERAGE:
cmake_args.append("-DONNX_COVERAGE=ON")
if COVERAGE or DEBUG:
# in order to get accurate coverage information, the
# build needs to turn off optimizations
build_type = "Debug"
cmake_args.append(f"-DCMAKE_BUILD_TYPE={build_type}")
if WINDOWS:
cmake_args.extend([
# we need to link with libpython on windows, so
# passing python version to window in order to
# find python in cmake
f"-DPY_VERSION={'{}.{}'.format(*sys.version_info[:2])}",
])
if USE_MSVC_STATIC_RUNTIME:
cmake_args.append("-DONNX_USE_MSVC_STATIC_RUNTIME=ON")
if platform.architecture()[0] == "64bit":
cmake_args.extend(["-A", "x64", "-T", "host=x64"])
else:
cmake_args.extend(["-A", "Win32", "-T", "host=x86"])
if ONNX_ML:
cmake_args.append("-DONNX_ML=1")
if ONNX_VERIFY_PROTO3:
cmake_args.append("-DONNX_VERIFY_PROTO3=1")
if ONNX_BUILD_TESTS:
cmake_args.append("-DONNX_BUILD_TESTS=ON")
if ONNX_DISABLE_EXCEPTIONS:
cmake_args.append("-DONNX_DISABLE_EXCEPTIONS=ON")
if "CMAKE_ARGS" in os.environ:
extra_cmake_args = shlex.split(os.environ["CMAKE_ARGS"])
# prevent crossfire with downstream scripts
del os.environ["CMAKE_ARGS"]
log.info(f"Extra cmake args: {extra_cmake_args}")
cmake_args.extend(extra_cmake_args)
cmake_args.append(TOP_DIR)
log.info(f"Using cmake args: {cmake_args}")
if "-DONNX_DISABLE_EXCEPTIONS=ON" in cmake_args:
raise RuntimeError(
"-DONNX_DISABLE_EXCEPTIONS=ON option is only available for c++ builds. Python binding require exceptions to be enabled."
)
subprocess.check_call(cmake_args)
build_args = [CMAKE, "--build", os.curdir]
if WINDOWS:
build_args.extend(["--config", build_type])
build_args.extend(["--", f"/maxcpucount:{self.jobs}"])
else:
build_args.extend(["--", "-j", str(self.jobs)])
subprocess.check_call(build_args)
def add_license_files(self):
license_files = self.distribution.metadata.license_files or []
for lf in license_files:
log.info("adding license file '%s'", lf)
pass
self.filelist.extend(license_files)
def info(self, msg, *args):
log.info(msg, *args)
def copy_file(src, dst, preserve_mode=1, preserve_times=1, update=0,
link=None, verbose=1, dry_run=0):
"""Copy a file 'src' to 'dst'. If 'dst' is a directory, then 'src' is
copied there with the same name; otherwise, it must be a filename. (If
the file exists, it will be ruthlessly clobbered.) If 'preserve_mode'
is true (the default), the file's mode (type and permission bits, or
whatever is analogous on the current platform) is copied. If
'preserve_times' is true (the default), the last-modified and
last-access times are copied as well. If 'update' is true, 'src' will
only be copied if 'dst' does not exist, or if 'dst' does exist but is
older than 'src'.
'link' allows you to make hard links (os.link) or symbolic links
(os.symlink) instead of copying: set it to "hard" or "sym"; if it is
None (the default), files are copied. Don't set 'link' on systems that
don't support it: 'copy_file()' doesn't check if hard or symbolic
linking is available. If hardlink fails, falls back to
_copy_file_contents().
Under Mac OS, uses the native file copy function in macostools; on
other systems, uses '_copy_file_contents()' to copy file contents.
Return a tuple (dest_name, copied): 'dest_name' is the actual name of
the output file, and 'copied' is true if the file was copied (or would
have been copied, if 'dry_run' true).
"""
# XXX if the destination file already exists, we clobber it if
# copying, but blow up if linking. Hmmm. And I don't know what
# macostools.copyfile() does. Should definitely be consistent, and
# should probably blow up if destination exists and we would be
# changing it (ie. it's not already a hard/soft link to src OR
# (not update) and (src newer than dst).
from distutils.dep_util import newer
from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE
if not os.path.isfile(src):
raise DistutilsFileError(
"can't copy '%s': doesn't exist or not a regular file" % src)
if os.path.isdir(dst):
dir = dst
dst = os.path.join(dst, os.path.basename(src))
else:
dir = os.path.dirname(dst)
if update and not newer(src, dst):
if verbose >= 1:
log.debug("not copying %s (output up-to-date)", src)
return (dst, 0)
try:
action = _copy_action[link]
except KeyError:
raise ValueError("invalid value '%s' for 'link' argument" % link)
if verbose >= 1:
if os.path.basename(dst) == os.path.basename(src):
log.info("%s %s -> %s", action, src, dir)
else:
log.info("%s %s -> %s", action, src, dst)
if dry_run:
return (dst, 1)
# If linking (hard or symbolic), use the appropriate system call
# (Unix only, of course, but that's the caller's responsibility)
elif link == 'hard':
if not (os.path.exists(dst) and os.path.samefile(src, dst)):
try:
os.link(src, dst)
return (dst, 1)
except OSError:
# If hard linking fails, fall back on copying file
# (some special filesystems don't support hard linking
# even under Unix, see issue #8876).
pass
elif link == 'sym':
if not (os.path.exists(dst) and os.path.samefile(src, dst)):
os.symlink(src, dst)
return (dst, 1)
# Otherwise (non-Mac, not linking), copy the file contents and
# (optionally) copy the times and mode.
_copy_file_contents(src, dst)
if preserve_mode or preserve_times:
st = os.stat(src)
# According to David Ascher <*****@*****.**>, utime() should be done
# before chmod() (at least under NT).
if preserve_times:
os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
if preserve_mode:
os.chmod(dst, S_IMODE(st[ST_MODE]))
return (dst, 1)
def run(self):
log.info("[pbr] Extracting deb version")
name = self.distribution.get_name()
print(version.VersionInfo(name).semantic_version().debian_string())
def parse_requirements(requirements_files=None, strip_markers=False):
if requirements_files is None:
requirements_files = get_requirements_files()
def egg_fragment(match):
# take a versioned egg fragment and return a
# versioned package requirement e.g.
# nova-1.2.3 becomes nova>=1.2.3
return re.sub(r'([\w.]+)-([\w.-]+)',
r'\1>=\2',
match.groups()[-1])
requirements = []
for line in get_reqs_from_files(requirements_files):
# Ignore comments
if (not line.strip()) or line.startswith('#'):
continue
# Ignore index URL lines
if re.match(r'^\s*(-i|--index-url|--extra-index-url).*', line):
continue
# Handle nested requirements files such as:
# -r other-requirements.txt
if line.startswith('-r'):
req_file = line.partition(' ')[2]
requirements += parse_requirements(
[req_file], strip_markers=strip_markers)
continue
try:
project_name = pkg_resources.Requirement.parse(line).project_name
except ValueError:
project_name = None
# For the requirements list, we need to inject only the portion
# after egg= so that distutils knows the package it's looking for
# such as:
# -e git://github.com/openstack/nova/master#egg=nova
# -e git://github.com/openstack/nova/master#egg=nova-1.2.3
# -e git+https://foo.com/zipball#egg=bar&subdirectory=baz
if re.match(r'\s*-e\s+', line):
line = re.sub(r'\s*-e\s+.*#egg=([^&]+).*$', egg_fragment, line)
# such as:
# http://github.com/openstack/nova/zipball/master#egg=nova
# http://github.com/openstack/nova/zipball/master#egg=nova-1.2.3
# git+https://foo.com/zipball#egg=bar&subdirectory=baz
elif re.match(r'\s*(https?|git(\+(https|ssh))?):', line):
line = re.sub(r'\s*(https?|git(\+(https|ssh))?):.*#egg=([^&]+).*$',
egg_fragment, line)
# -f lines are for index locations, and don't get used here
elif re.match(r'\s*-f\s+', line):
line = None
reason = 'Index Location'
if line is not None:
line = re.sub('#.*$', '', line)
if strip_markers:
semi_pos = line.find(';')
if semi_pos < 0:
semi_pos = None
line = line[:semi_pos]
requirements.append(line)
else:
log.info(
'[pbr] Excluding %s: %s' % (project_name, reason))
return requirements
def run (self):
if not self.skip_build:
self.run_command('build')
install = self.reinitialize_command('install', reinit_subcommands=1)
install.prefix = self.bdist_dir
install.skip_build = self.skip_build
install.warn_dir = 0
install_lib = self.reinitialize_command('install_lib')
# we do not want to include pyc or pyo files
install_lib.compile = 0
install_lib.optimize = 0
if self.distribution.has_ext_modules():
# If we are building an installer for a Python version other
# than the one we are currently running, then we need to ensure
# our build_lib reflects the other Python version rather than ours.
# Note that for target_version!=sys.version, we must have skipped the
# build step, so there is no issue with enforcing the build of this
# version.
target_version = self.target_version
if not target_version:
assert self.skip_build, "Should have already checked this"
target_version = sys.version[0:3]
plat_specifier = ".%s-%s" % (self.plat_name, target_version)
build = self.get_finalized_command('build')
build.build_lib = os.path.join(build.build_base,
'lib' + plat_specifier)
log.info("installing to %s", self.bdist_dir)
install.ensure_finalized()
# avoid warning of 'install_lib' about installing
# into a directory not in sys.path
sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))
install.run()
del sys.path[0]
self.mkpath(self.dist_dir)
fullname = self.distribution.get_fullname()
installer_name = self.get_installer_filename(fullname)
installer_name = os.path.abspath(installer_name)
if os.path.exists(installer_name): os.unlink(installer_name)
metadata = self.distribution.metadata
author = metadata.author
if not author:
author = metadata.maintainer
if not author:
author = "UNKNOWN"
version = metadata.get_version()
# ProductVersion must be strictly numeric
# XXX need to deal with prerelease versions
sversion = "%d.%d.%d" % StrictVersion(version).version
# Prefix ProductName with Python x.y, so that
# it sorts together with the other Python packages
# in Add-Remove-Programs (APR)
fullname = self.distribution.get_fullname()
if self.target_version:
product_name = "Python %s %s" % (self.target_version, fullname)
else:
product_name = "Python %s" % (fullname)
self.db = msilib.init_database(installer_name, schema,
product_name, msilib.gen_uuid(),
sversion, author)
msilib.add_tables(self.db, sequence)
props = [('DistVersion', version)]
email = metadata.author_email or metadata.maintainer_email
if email:
props.append(("ARPCONTACT", email))
if metadata.url:
props.append(("ARPURLINFOABOUT", metadata.url))
if props:
add_data(self.db, 'Property', props)
self.add_find_python()
self.add_files()
self.add_scripts()
self.add_ui()
self.db.Commit()
if hasattr(self.distribution, 'dist_files'):
tup = 'bdist_msi', self.target_version or 'any', fullname
self.distribution.dist_files.append(tup)
if not self.keep_temp:
remove_tree(self.bdist_dir, dry_run=self.dry_run)
def add_to_root(fname: Path):
log.info(' adding %s', fname.name)
archive.write(str(fname), fname.name)
def byte_compile(py_files,
optimize=0,
force=0,
prefix=None,
base_dir=None,
verbose=1,
dry_run=0,
direct=None):
"""Byte-compile a collection of Python source files to either .pyc
or .pyo files in the same directory. 'py_files' is a list of files
to compile; any files that don't end in ".py" are silently skipped.
'optimize' must be one of the following:
0 - don't optimize (generate .pyc)
1 - normal optimization (like "python -O")
2 - extra optimization (like "python -OO")
If 'force' is true, all files are recompiled regardless of
timestamps.
The source filename encoded in each bytecode file defaults to the
filenames listed in 'py_files'; you can modify these with 'prefix' and
'basedir'. 'prefix' is a string that will be stripped off of each
source filename, and 'base_dir' is a directory name that will be
prepended (after 'prefix' is stripped). You can supply either or both
(or neither) of 'prefix' and 'base_dir', as you wish.
If 'dry_run' is true, doesn't actually do anything that would
affect the filesystem.
Byte-compilation is either done directly in this interpreter process
with the standard py_compile module, or indirectly by writing a
temporary script and executing it. Normally, you should let
'byte_compile()' figure out to use direct compilation or not (see
the source for details). The 'direct' flag is used by the script
generated in indirect mode; unless you know what you're doing, leave
it set to None.
"""
# nothing is done if sys.dont_write_bytecode is True
if sys.dont_write_bytecode:
raise DistutilsByteCompileError('byte-compiling is disabled.')
# First, if the caller didn't force us into direct or indirect mode,
# figure out which mode we should be in. We take a conservative
# approach: choose direct mode *only* if the current interpreter is
# in debug mode and optimize is 0. If we're not in debug mode (-O
# or -OO), we don't know which level of optimization this
# interpreter is running with, so we can't do direct
# byte-compilation and be certain that it's the right thing. Thus,
# always compile indirectly if the current interpreter is in either
# optimize mode, or if either optimization level was requested by
# the caller.
if direct is None:
direct = (__debug__ and optimize == 0)
# "Indirect" byte-compilation: write a temporary script and then
# run it with the appropriate flags.
if not direct:
try:
from tempfile import mkstemp
(script_fd, script_name) = mkstemp(".py")
except ImportError:
from tempfile import mktemp
(script_fd, script_name) = None, mktemp(".py")
log.info("writing byte-compilation script '%s'", script_name)
if not dry_run:
if script_fd is not None:
script = os.fdopen(script_fd, "w")
else:
script = open(script_name, "w")
script.write("""\
from distutils.util import byte_compile
files = [
""")
# XXX would be nice to write absolute filenames, just for
# safety's sake (script should be more robust in the face of
# chdir'ing before running it). But this requires abspath'ing
# 'prefix' as well, and that breaks the hack in build_lib's
# 'byte_compile()' method that carefully tacks on a trailing
# slash (os.sep really) to make sure the prefix here is "just
# right". This whole prefix business is rather delicate -- the
# problem is that it's really a directory, but I'm treating it
# as a dumb string, so trailing slashes and so forth matter.
#py_files = map(os.path.abspath, py_files)
#if prefix:
# prefix = os.path.abspath(prefix)
script.write(string.join(map(repr, py_files), ",\n") + "]\n")
script.write("""
byte_compile(files, optimize=%r, force=%r,
prefix=%r, base_dir=%r,
verbose=%r, dry_run=0,
direct=1)
""" % (optimize, force, prefix, base_dir, verbose))
script.close()
cmd = [sys.executable, script_name]
if optimize == 1:
cmd.insert(1, "-O")
elif optimize == 2:
cmd.insert(1, "-OO")
spawn(cmd, dry_run=dry_run)
execute(os.remove, (script_name, ),
"removing %s" % script_name,
dry_run=dry_run)
# "Direct" byte-compilation: use the py_compile module to compile
# right here, right now. Note that the script generated in indirect
# mode simply calls 'byte_compile()' in direct mode, a weird sort of
# cross-process recursion. Hey, it works!
else:
from py_compile import compile
for file in py_files:
if file[-3:] != ".py":
# This lets us be lazy and not filter filenames in
# the "install_lib" command.
continue
# Terminology from the py_compile module:
# cfile - byte-compiled file
# dfile - purported source filename (same as 'file' by default)
cfile = file + (__debug__ and "c" or "o")
dfile = file
if prefix:
if file[:len(prefix)] != prefix:
raise ValueError, \
("invalid prefix: filename %r doesn't start with %r"
% (file, prefix))
dfile = dfile[len(prefix):]
if base_dir:
dfile = os.path.join(base_dir, dfile)
cfile_base = os.path.basename(cfile)
if direct:
if force or newer(file, cfile):
log.info("byte-compiling %s to %s", file, cfile_base)
if not dry_run:
compile(file, cfile, dfile)
else:
log.debug("skipping byte-compilation of %s to %s", file,
cfile_base)
def delete_file(self, filename):
"""Delete `filename` (if not a dry run) after announcing it"""
log.info("deleting %s", filename)
if not self.dry_run:
os.unlink(filename)
def run(self):
# Generate metadata first
self.run_command("egg_info")
# We run install_lib before install_data, because some data hacks
# pull their data path from the install_lib command.
log.info("installing library code to %s", self.bdist_dir)
instcmd = self.get_finalized_command('install')
old_root = instcmd.root
instcmd.root = None
if self.distribution.has_c_libraries() and not self.skip_build:
self.run_command('build_clib')
cmd = self.call_command('install_lib', warn_dir=0)
instcmd.root = old_root
all_outputs, ext_outputs = self.get_ext_outputs()
self.stubs = []
to_compile = []
for (p, ext_name) in enumerate(ext_outputs):
filename, ext = os.path.splitext(ext_name)
pyfile = os.path.join(self.bdist_dir, strip_module(filename) +
'.py')
self.stubs.append(pyfile)
log.info("creating stub loader for %s", ext_name)
if not self.dry_run:
write_stub(os.path.basename(ext_name), pyfile)
to_compile.append(pyfile)
ext_outputs[p] = ext_name.replace(os.sep, '/')
if to_compile:
cmd.byte_compile(to_compile)
if self.distribution.data_files:
self.do_install_data()
# Make the EGG-INFO directory
archive_root = self.bdist_dir
egg_info = os.path.join(archive_root, 'EGG-INFO')
self.mkpath(egg_info)
if self.distribution.scripts:
script_dir = os.path.join(egg_info, 'scripts')
log.info("installing scripts to %s", script_dir)
self.call_command('install_scripts', install_dir=script_dir,
no_ep=1)
self.copy_metadata_to(egg_info)
native_libs = os.path.join(egg_info, "native_libs.txt")
if all_outputs:
log.info("writing %s", native_libs)
if not self.dry_run:
ensure_directory(native_libs)
libs_file = open(native_libs, 'wt')
libs_file.write('\n'.join(all_outputs))
libs_file.write('\n')
libs_file.close()
elif os.path.isfile(native_libs):
log.info("removing %s", native_libs)
if not self.dry_run:
os.unlink(native_libs)
write_safety_flag(
os.path.join(archive_root, 'EGG-INFO'), self.zip_safe()
)
if os.path.exists(os.path.join(self.egg_info, 'depends.txt')):
log.warn(
"WARNING: 'depends.txt' will not be used by setuptools 0.6!\n"
"Use the install_requires/extras_require setup() args instead."
)
if self.exclude_source_files:
self.zap_pyfiles()
# Make the archive
make_zipfile(self.egg_output, archive_root, verbose=self.verbose,
dry_run=self.dry_run, mode=self.gen_header())
if not self.keep_temp:
remove_tree(self.bdist_dir, dry_run=self.dry_run)
# Add to 'Distribution.dist_files' so that the "upload" command works
getattr(self.distribution, 'dist_files', []).append(
('bdist_egg', get_python_version(), self.egg_output))
def move_file (src, dst,
verbose=1,
dry_run=0):
"""Move a file 'src' to 'dst'. If 'dst' is a directory, the file will
be moved into it with the same name; otherwise, 'src' is just renamed
to 'dst'. Return the new full name of the file.
Handles cross-device moves on Unix using 'copy_file()'. What about
other systems???
"""
from os.path import exists, isfile, isdir, basename, dirname
import errno
if verbose >= 1:
log.info("moving %s -> %s", src, dst)
if dry_run:
return dst
if not isfile(src):
raise DistutilsFileError("can't move '%s': not a regular file" % src)
if isdir(dst):
dst = os.path.join(dst, basename(src))
elif exists(dst):
raise DistutilsFileError(
"can't move '%s': destination '%s' already exists" %
(src, dst))
if not isdir(dirname(dst)):
raise DistutilsFileError(
"can't move '%s': destination '%s' not a valid path" %
(src, dst))
copy_it = False
try:
os.rename(src, dst)
except os.error as e:
(num, msg) = e
if num == errno.EXDEV:
copy_it = True
else:
raise DistutilsFileError(
"couldn't move '%s' to '%s': %s" % (src, dst, msg))
if copy_it:
copy_file(src, dst, verbose=verbose)
try:
os.unlink(src)
except os.error as e:
(num, msg) = e
try:
os.unlink(dst)
except os.error:
pass
raise DistutilsFileError(
"couldn't move '%s' to '%s' by copy/delete: "
"delete '%s' failed: %s"
% (src, dst, src, msg))
return dst
def classifiers(self):
''' Fetch the list of classifiers from the server.
'''
response = urllib2.urlopen(self.repository+'?:action=list_classifiers')
log.info(response.read())
(src, errstr))
if not dry_run:
mkpath(dst)
outputs = []
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if (condition is not None) and (not condition(src_name)):
continue
if preserve_symlinks and os.path.islink(src_name):
link_dest = os.readlink(src_name)
log.info("linking %s -> %s", dst_name, link_dest)
if not dry_run:
if update and not newer(src, dst_name):
pass
else:
if os.path.islink(dst_name):
os.remove(dst_name)
os.symlink(link_dest, dst_name)
outputs.append(dst_name)
elif os.path.isdir(src_name):
outputs.extend(
copy_tree(src_name,
dst_name,
preserve_mode,
preserve_times,
def run(self):
import os
global wmmexec
if wmmexec[-13:] == "wolframscript":
with open("wolfram_kernel/wmath.in") as f:
script = f.read()
with open("wolfram_kernel/wmath", "w") as f:
f.write("#!" + wmmexec + " -c\n")
f.write(script)
if platform.system() == "Linux":
os.chmod("wolfram_kernel/wmath", 0o755)
wmmexec = (setuptools.__path__)[0][:-10] + "wolfram_kernel/wmath"
user = '******' in sys.argv or not _is_root()
configfilestr = "# iwolfram configuration file\nmathexec = '{wolfram-caller-script-path}'\n\n"
configfilestr = configfilestr.replace('{wolfram-caller-script-path}',
wmmexec)
with open('wolfram_kernel/config.py', 'w') as f:
f.write(configfilestr)
#Run the standard intallation
install.run(self)
def install_kernelspec(self):
from ipykernel.kernelspec import write_kernel_spec
from jupyter_client.kernelspec import KernelSpecManager
from wolfram_kernel.wolfram_kernel import WolframKernel
kernel_json = WolframKernel.kernel_json
kernel_js = WolframKernel.kernel_js
kernel_spec_manager = KernelSpecManager()
log.info('Writing kernel spec')
kernel_spec_path = write_kernel_spec(overrides=kernel_json)
with open(kernel_spec_path + "/kernel.js", "w") as jsfile:
jsfile.write(kernel_js)
log.info('Installing kernel spec')
try:
kernel_spec_manager.install_kernel_spec(
kernel_spec_path,
kernel_name=kernel_json['name'],
user=user)
except:
log.error('Failed to install kernel spec in ' +
kernel_spec_path)
kernel_spec_manager.install_kernel_spec(
kernel_spec_path,
kernel_name=kernel_json['name'],
user=not user)
print("Installing kernel spec")
#Build and Install the kernelspec
install_kernelspec(self)
log.info("Installing nbextension")
from notebook.nbextensions import install_nbextension
import os.path
try:
install_nbextension(
os.path.join(os.path.dirname(__file__), 'nbmathics'),
overwrite=True,
)
jup_nbex_exec = os.path.dirname(
sys.executable) + "/" + "jupyter-nbextension"
os.system(jup_nbex_exec + " install --system nbmathics")
os.system(jup_nbex_exec + " enable --system --py nbmathics")
except:
log.info("nbextension can not be installed")
def verify_metadata(self):
''' Send the metadata to the package index server to be checked.
'''
# send the info to the server and report the result
(code, result) = self.post_to_server(self.build_post_data('verify'))
log.info('Server response (%s): %s' % (code, result))
def run(self):
# base install
_install.run(self)
# custom install of backend
log.info('Now installing cppyy-cling into cppyy_backend')
builddir = get_builddir()
if not os.path.exists(builddir):
raise DistutilsSetupError('Failed to find build dir!')
# use $MAKE to install if it is defined
env_make = os.getenv("MAKE")
if not env_make:
install_cmd = 'cmake'
install_args = [
'--build', '.', '--config',
get_build_type(), '--target', 'install'
]
else:
install_args = env_make.split()
install_cmd, install_args = install_args[0], install_args[1:] + [
'install'
]
prefix = get_prefix()
log.info('Now creating installation under %s ...', prefix)
if env_make: os.unsetenv("MAKE")
if subprocess.call([install_cmd] + install_args, cwd=builddir) != 0:
raise DistutilsSetupError('Failed to install cppyy-cling')
if env_make: os.putenv("MAKE", env_make)
if 'linux' in sys.platform:
# remove allDict.cxx.pch as it's not portable (rebuild on first run, see cppyy)
log.info('removing allDict.cxx.pch')
os.remove(os.path.join(get_prefix(), 'etc', 'allDict.cxx.pch'))
# for manylinux, reset the default cxxversion to 17 if no user override
if not 'STDCXX' in os.environ and is_manylinux():
log.info('updating root-config to C++17 for manylinux')
inp = os.path.join(get_prefix(), 'bin', 'root-config')
outp = inp + '.new'
outfile = open(outp, 'w')
for line in open(inp).readlines():
if 'cxxversion=' == line[:11]:
line = 'cxxversion=cxx17\n'
outfile.write(line)
outfile.close()
os.rename(outp, inp)
os.chmod(
inp,
stat.S_IMODE(os.lstat(inp).st_mode) | stat.S_IXUSR
| stat.S_IXGRP | stat.S_IXOTH)
log.info('updating allCppflags.txt to C++17 for manylinux')
inp = os.path.join(get_prefix(), 'etc', 'dictpch',
'allCppflags.txt')
outp = inp + '.new'
outfile = open(outp, 'w')
for line in open(inp).readlines():
if '-std=' == line[:5]:
line = '-std=c++1z\n'
outfile.write(line)
outfile.close()
os.rename(outp, inp)
install_path = self._get_install_path()
log.info('Copying installation to: %s ...', install_path)
self.copy_tree(os.path.join(get_prefix(), os.path.pardir),
install_path)
log.info('Install finished')
def send_metadata(self):
''' Send the metadata to the package index server.
Well, do the following:
1. figure who the user is, and then
2. send the data as a Basic auth'ed POST.
First we try to read the username/password from $HOME/.pypirc,
which is a ConfigParser-formatted file with a section
[distutils] containing username and password entries (both
in clear text). Eg:
[distutils]
index-servers =
pypi
[pypi]
username: fred
password: sekrit
Otherwise, to figure who the user is, we offer the user three
choices:
1. use existing login,
2. register as a new user, or
3. set the password to a random string and email the user.
'''
# see if we can short-cut and get the username/password from the
# config
if self.has_config:
choice = '1'
username = self.username
password = self.password
else:
choice = 'x'
username = password = ''
# get the user's login info
choices = '1 2 3 4'.split()
while choice not in choices:
self.announce('''\
We need to know who you are, so please choose either:
1. use your existing login,
2. register as a new user,
3. have the server generate a new password for you (and email it to you), or
4. quit
Your selection [default 1]: ''', log.INFO)
choice = raw_input()
if not choice:
choice = '1'
elif choice not in choices:
print 'Please choose one of the four options!'
if choice == '1':
# get the username and password
while not username:
username = raw_input('Username: '******'Password: '******'submit'),
auth)
self.announce('Server response (%s): %s' % (code, result),
log.INFO)
# possibly save the login
if code == 200:
if self.has_config:
# sharing the password in the distribution instance
# so the upload command can reuse it
self.distribution.password = password
else:
self.announce(('I can store your PyPI login so future '
'submissions will be faster.'), log.INFO)
self.announce('(the login will be stored in %s)' % \
self._get_rc_file(), log.INFO)
choice = 'X'
while choice.lower() not in 'yn':
choice = raw_input('Save your login (y/N)?')
if not choice:
choice = 'n'
if choice.lower() == 'y':
self._store_pypirc(username, password)
elif choice == '2':
data = {':action': 'user'}
data['name'] = data['password'] = data['email'] = ''
data['confirm'] = None
while not data['name']:
data['name'] = raw_input('Username: '******'password'] != data['confirm']:
while not data['password']:
data['password'] = getpass.getpass('Password: '******'confirm']:
data['confirm'] = getpass.getpass(' Confirm: ')
if data['password'] != data['confirm']:
data['password'] = ''
data['confirm'] = None
print "Password and confirm don't match!"
while not data['email']:
data['email'] = raw_input(' EMail: ')
code, result = self.post_to_server(data)
if code != 200:
log.info('Server response (%s): %s' % (code, result))
else:
log.info('You will receive an email shortly.')
log.info(('Follow the instructions in it to '
'complete registration.'))
elif choice == '3':
data = {':action': 'password_reset'}
data['email'] = ''
while not data['email']:
data['email'] = raw_input('Your email address: ')
code, result = self.post_to_server(data)
log.info('Server response (%s): %s' % (code, result))
def init_msvc_env(platform_arch, build_type):
from distutils.msvc9compiler import VERSION as MSVC_VERSION
log.info("Searching MSVC compiler version {}".format(MSVC_VERSION))
vcdir_path = find_vcdir(MSVC_VERSION)
if not vcdir_path:
raise DistutilsSetupError(
"Failed to find the MSVC compiler version {} on your "
"system.".format(MSVC_VERSION))
else:
log.info("Found {}".format(vcdir_path))
log.info("Searching MSVC compiler {} environment init script".format(
MSVC_VERSION))
if platform_arch.startswith("32"):
vcvars_path = os.path.join(vcdir_path, "bin", "vcvars32.bat")
else:
vcvars_path = os.path.join(vcdir_path, "bin", "vcvars64.bat")
if not os.path.exists(vcvars_path):
vcvars_path = os.path.join(vcdir_path, "bin", "amd64",
"vcvars64.bat")
if not os.path.exists(vcvars_path):
vcvars_path = os.path.join(vcdir_path, "bin", "amd64",
"vcvarsamd64.bat")
if not os.path.exists(vcvars_path):
# MSVC init script not found, try to find and init Windows SDK env
log.error("Failed to find the MSVC compiler environment init script "
"(vcvars.bat) on your system.")
winsdk_setenv(platform_arch, build_type)
return
else:
log.info("Found {}".format(vcvars_path))
# Get MSVC env
log.info("Using MSVC {} in {}".format(MSVC_VERSION, vcvars_path))
msvc_arch = "x86" if platform_arch.startswith("32") else "amd64"
log.info("Getting MSVC env for {} architecture".format(msvc_arch))
vcvars_cmd = [vcvars_path, msvc_arch]
msvc_env = get_environment_from_batch_command(vcvars_cmd)
msvc_env_paths = os.pathsep.join([
msvc_env[k] for k in msvc_env if k.upper() == 'PATH'
]).split(os.pathsep)
msvc_env_without_paths = dict([(k, msvc_env[k]) for k in msvc_env
if k.upper() != 'PATH'])
# Extend os.environ with MSVC env
log.info("Initializing MSVC env...")
update_env_path(msvc_env_paths)
for k in sorted(msvc_env_without_paths):
v = msvc_env_without_paths[k]
log.info("Inserting '{} = {}' to environment".format(k, v))
os.environ[k] = v
log.info("Done initializing MSVC env")
def run(self):
# base run
_build.run(self)
# custom run
log.info('Now building cppyy-cling')
builddir = get_builddir()
prefix = get_prefix()
srcdir = get_srcdir()
if not os.path.exists(srcdir):
log.info(
'No src directory ... creating with "python create_src_directory.py"'
)
if subprocess.call(['python', 'create_src_directory.py']) != 0:
log.error('ERROR: the source directory "%s" does not exist' %
srcdir)
log.error('Please run "python create_src_directory.py" first.')
sys.exit(1)
if not os.path.exists(builddir):
log.info('Creating build directory %s ...' % builddir)
os.makedirs(builddir)
# get C++ standard to use, if set
try:
stdcxx = os.environ['STDCXX']
except KeyError:
if is_manylinux():
stdcxx = '11'
elif 'win32' in sys.platform:
stdcxx = '14' # current cmake claims MSVC'17 does not support C++17 yet
else:
stdcxx = '17'
if not stdcxx in ['11', '14', '17']:
log.fatal('FATAL: envar STDCXX should be one of 11, 14, or 17')
sys.exit(1)
stdcxx = '-Dcxx' + stdcxx + '=ON'
# extra optimization flags for Cling
if not 'EXTRA_CLING_ARGS' in os.environ:
has_avx = False
if not is_manylinux():
try:
for line in open('/proc/cpuinfo', 'r'):
if 'avx' in line:
has_avx = True
break
except Exception:
try:
cli_arg = subprocess.check_output(
['sysctl', 'machdep.cpu.features'])
has_avx = 'avx' in cli_arg.decode(
"utf-8").strip().lower()
except Exception:
pass
extra_args = '-O2'
if has_avx: extra_args += ' -mavx'
os.putenv('EXTRA_CLING_ARGS', extra_args)
CMAKE_COMMAND = [
'cmake', srcdir, stdcxx, '-DLLVM_ENABLE_TERMINFO=0',
'-Dminimal=ON', '-Dasimage=OFF', '-Droot7=OFF', '-Dhttp=OFF',
'-Dbuiltin_pcre=ON', '-Dbuiltin_freetype=ON', '-Dbuiltin_zlib=ON',
'-Dbuiltin_xxhash=ON'
]
if 'darwin' in sys.platform:
CMAKE_COMMAND.append('-Dlibcxx=ON')
CMAKE_COMMAND.append('-DCMAKE_BUILD_TYPE=' + get_build_type())
if 'win32' in sys.platform:
import platform
if '64' in platform.architecture()[0]:
CMAKE_COMMAND += [
'-Thost=x64', '-DCMAKE_GENERATOR_PLATFORM=x64',
'-Dall=OFF', '-Dmathmore=OFF', '-Dbuiltin_ftgl=OFF',
'-Droofit=OFF', '-Dgfal=OFF', '-Dfftw3=OFF'
]
FFTW_INC = os.environ.get("FFTW_INC", None)
FFTW_LIB = os.environ.get("FFTW_LIB", None)
if FFTW_INC and FFTW_LIB:
CMAKE_COMMAND += [
"-DFFTW_INCLUDE_DIR={}".format(FFTW_INC),
"-DFFTW_LIBRARY={}".format(FFTW_LIB)
]
else:
CMAKE_COMMAND += ['-Dbuiltin_freetype=OFF']
CMAKE_COMMAND.append('-DCMAKE_INSTALL_PREFIX=' + prefix)
log.info('Running cmake for cppyy-cling: %s', ' '.join(CMAKE_COMMAND))
if subprocess.call(CMAKE_COMMAND, cwd=builddir) != 0:
raise DistutilsSetupError('Failed to configure cppyy-cling')
# use $MAKE to build if it is defined
env_make = os.getenv('MAKE')
if not env_make:
build_cmd = 'cmake'
# default to using all available cores (x2 if hyperthreading enabled)
nprocs = os.getenv("MAKE_NPROCS") or '0'
try:
nprocs = int(nprocs)
except ValueError:
log.warn(
"Integer expected for MAKE_NPROCS, but got %s (ignored)",
nprocs)
nprocs = 0
if nprocs < 1:
nprocs = multiprocessing.cpu_count()
build_args = ['--build', '.', '--config', get_build_type(), '--']
if 'win32' in sys.platform:
build_args.append('/maxcpucount:' + str(nprocs))
else:
build_args.append('-j' + str(nprocs))
else:
build_args = env_make.split()
build_cmd, build_args = build_args[0], build_args[1:]
log.info('Now building cppyy-cling and dependencies ...')
if env_make: os.unsetenv("MAKE")
if subprocess.call([build_cmd] + build_args, cwd=builddir) != 0:
raise DistutilsSetupError('Failed to build cppyy-cling')
if env_make: os.putenv('MAKE', env_make)
log.info('Build finished')
def log_message(self, msg, *args):
log.info(msg, *args)
def copydir(src,
dst,
filter=None,
ignore=None,
force=True,
recursive=True,
vars=None,
dir_filter_function=None,
file_filter_function=None,
force_copy_symlinks=False):
if vars is not None:
src = src.format(**vars)
dst = dst.format(**vars)
if filter is not None:
for i in range(len(filter)):
filter[i] = filter[i].format(**vars)
if ignore is not None:
for i in range(len(ignore)):
ignore[i] = ignore[i].format(**vars)
if not os.path.exists(src) and not force:
log.info("**Skiping copy tree {} to {}. Source does not exists. "
"filter={}. ignore={}.".format(src, dst, filter, ignore))
return []
log.info("Copying tree {} to {}. filter={}. ignore={}.".format(
src, dst, filter, ignore))
names = os.listdir(src)
results = []
errors = []
for name in names:
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
try:
if os.path.isdir(srcname):
if (dir_filter_function
and not dir_filter_function(name, src, srcname)):
continue
if recursive:
results.extend(
copydir(srcname, dstname, filter, ignore, force,
recursive, vars, dir_filter_function,
file_filter_function, force_copy_symlinks))
else:
if ((file_filter_function is not None
and not file_filter_function(name, srcname)) or
(filter is not None and not filter_match(name, filter)) or
(ignore is not None and filter_match(name, ignore))):
continue
if not os.path.exists(dst):
os.makedirs(dst)
results.append(
copyfile(srcname, dstname, True, vars,
force_copy_symlinks))
# catch the Error from the recursive copytree so that we can
# continue with other files
except shutil.Error as err:
errors.extend(err.args[0])
except EnvironmentError as why:
errors.append((srcname, dstname, str(why)))
try:
if os.path.exists(dst):
shutil.copystat(src, dst)
except OSError as why:
if WindowsError is not None and isinstance(why, WindowsError):
# Copying file access times may fail on Windows
pass
else:
errors.extend((src, dst, str(why)))
if errors:
raise EnvironmentError(errors)
return results
])
def collect_static_paths_from(static_folder_path):
static_files = []
for a_path in os.listdir(static_folder_path):
static_files.append(os.path.join(static_folder_path, a_path))
return static_files
static_path = os.path.join('.', 'ipyleaflet_reset', 'static')
static_files = collect_static_paths_from(static_path)
log.set_verbosity(log.DEBUG)
log.info('setup.py entered')
log.info('$PATH=%s' % os.environ['PATH'])
LONG_DESCRIPTION = 'a jupyter leaflet control to reset the map'
def js_prerelease(command, strict=False):
"""decorator for building minified js/css prior to another command"""
class DecoratedCommand(command):
def run(self):
jsdeps = self.distribution.get_command_obj('jsdeps')
if not is_repo and all(os.path.exists(t) for t in jsdeps.targets):
# sdist, nothing to do
command.run(self)
return
def winsdk_setenv(platform_arch, build_type):
from distutils.msvc9compiler import VERSION as MSVC_VERSION
from distutils.msvc9compiler import Reg
from distutils.msvc9compiler import HKEYS
from distutils.msvc9compiler import WINSDK_BASE
sdk_version_map = {
"v6.0a": 9.0,
"v6.1": 9.0,
"v7.0": 9.0,
"v7.0a": 10.0,
"v7.1": 10.0
}
log.info("Searching Windows SDK with MSVC compiler version {}".format(
MSVC_VERSION))
setenv_paths = []
for base in HKEYS:
sdk_versions = Reg.read_keys(base, WINSDK_BASE)
if sdk_versions:
for sdk_version in sdk_versions:
installationfolder = Reg.get_value(
WINSDK_BASE + "\\" + sdk_version, "installationfolder")
productversion = Reg.get_value(
WINSDK_BASE + "\\" + sdk_version, "productversion")
setenv_path = os.path.join(installationfolder,
os.path.join('bin', 'SetEnv.cmd'))
if not os.path.exists(setenv_path):
continue
if not sdk_version in sdk_version_map:
continue
if sdk_version_map[sdk_version] != MSVC_VERSION:
continue
setenv_paths.append(setenv_path)
if len(setenv_paths) == 0:
raise DistutilsSetupError(
"Failed to find the Windows SDK with MSVC compiler "
"version {}".format(MSVC_VERSION))
for setenv_path in setenv_paths:
log.info("Found {}".format(setenv_path))
# Get SDK env (use latest SDK version installed on system)
setenv_path = setenv_paths[-1]
log.info("Using {} ".format(setenv_path))
build_arch = "/x86" if platform_arch.startswith("32") else "/x64"
build_type = "/Debug" if build_type.lower() == "debug" else "/Release"
setenv_cmd = [setenv_path, build_arch, build_type]
setenv_env = get_environment_from_batch_command(setenv_cmd)
setenv_env_paths = os.pathsep.join([
setenv_env[k] for k in setenv_env if k.upper() == 'PATH'
]).split(os.pathsep)
setenv_env_without_paths = dict([(k, setenv_env[k]) for k in setenv_env
if k.upper() != 'PATH'])
# Extend os.environ with SDK env
log.info("Initializing Windows SDK env...")
update_env_path(setenv_env_paths)
for k in sorted(setenv_env_without_paths):
v = setenv_env_without_paths[k]
log.info("Inserting '{} = {}' to environment".format(k, v))
os.environ[k] = v
log.info("Done initializing Windows SDK env")
