def _do_upgrade(self, dist):
# Build up a requirement for a higher bugfix release but a lower minor
# release (so API compatibility is guaranteed)
next_version = _next_version(dist.parsed_version)
req = pkg_resources.Requirement.parse(
'{0}>{1},<{2}'.format(DIST_NAME, dist.version, next_version))
package_index = PackageIndex(index_url=self.index_url)
upgrade = package_index.obtain(req)
if upgrade is not None:
return self._do_download(version=upgrade.version)
def _do_upgrade(self, dist):
# Build up a requirement for a higher bugfix release but a lower minor
# release (so API compatibility is guaranteed)
next_version = _next_version(dist.parsed_version)
req = pkg_resources.Requirement.parse('{0}>{1},<{2}'.format(
DIST_NAME, dist.version, next_version))
package_index = PackageIndex(index_url=self.index_url)
upgrade = package_index.obtain(req)
if upgrade is not None:
return self._do_download(version=upgrade.version)
def _do_upgrade(dist, index_url):
# Build up a requirement for a higher bugfix release but a lower minor
# release (so API compatibility is guaranteed)
# sketchy version parsing--maybe come up with something a bit more
# robust for this
major, minor = (int(part) for part in dist.parsed_version[:2])
next_minor = '.'.join([str(major), str(minor + 1), '0'])
req = pkg_resources.Requirement.parse(
'{0}>{1},<{2}'.format(DIST_NAME, dist.version, next_minor))
package_index = PackageIndex(index_url=index_url)
upgrade = package_index.obtain(req)
if upgrade is not None:
return _do_download(version=upgrade.version, index_url=index_url)
class bdist_pkg(Command):
description = 'create FreeBSD pkg distribution'
user_options = [
('bdist-base=', 'b',
'Base directory for creating built distributions.'),
('dist-dir=', 'd',
'Directory to put distribute files in.'),
('format=', 'f',
'Set format as the package output format. It can be one'
' of txz, tbz, tgz or tar. If an invalid or no format is specified'
' tgz is assumed.'),
('keep-temp', None,
'Keep intermediate build directories and files.'),
('origin=', None,
'Custom origin name for build package.'),
('use-pypi-deps', None,
'Automatically convert unknown Python dependencies to package ones.'
' Note that those dependencies will be named with py{}{}- prefix and'
' assumes that you have such packages in repository.'
''.format(*sys.version_info[:2])),
('use-wheel', None,
'Use bdist_wheel to generated install layout instead of install'
' command.'),
('with-py-prefix', None,
'Prepends py{}{}- prefix to package name.'
''.format(*sys.version_info[:2])),
]
boolean_options = ('keep-temp', 'use-wheel', 'python-deps-to-pkg',
'with-py-prefix')
compressor_for_format = {
'txz': lzma,
'tgz': gzip,
'tbz': bz2,
}
def initialize_options(self):
self.bdist_base = None
self.dist_dir = None
self.format = None
self.keep_temp = False
self.name_prefix = None
self.package_index = PackageIndex()
self.requirements_mapping = None
self.selected_options = None
self.use_pypi_deps = False
self.use_wheel = False
self.with_py_prefix = False
self.initialize_manifest_options()
def initialize_manifest_options(self):
# TODO: What is it and how to use it?
# self.annotations = None
self.abi = None
self.arch = None
self.categories = None
# TODO: Could conflicts be useful for us?
# self.conflicts = None
self.comment = None
# TODO: What is it and how to use it?
# self.dep_formula = None
self.deps = None
self.desc = None
# These fields are autogenerated:
# self.directories = None
# self.dirs = None
# self.files = None
# self.flatsize = None
self.groups = None
self.license = None
self.maintainer = None
# TODO: should that be single message or multiple ones?
# self.messages = None
self.name = None
self.options = None
self.selected_options = None
# Since we use extras, which don't have either defaults or descriptions
# these fields are not supported so far:
# self.options_defaults = None
# self.options_descriptions = None
self.origin = None
# TODO: What is the path?
# self.path = None
self.prefix = None
self.provides = None
self.requires = None
self.scripts = None
# TODO: Do we need shared libs support?
# self.shlibs = None
# self.shlibs_provides = None
# self.shlibs_requires = None
# TODO: Support checksum.
# self.sum = None
self.users = None
self.version = None
# TODO: Can Python packages be vital?
# self.vital = None
self.www = None
def finalize_options(self):
self.set_undefined_options('bdist', ('bdist_base', 'bdist_base'))
self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))
self.ensure_format('tgz')
self.bdist_dir = os.path.join(self.bdist_base, 'pkg')
self.install_dir = os.path.join(self.bdist_dir, 'root')
self.finalize_manifest_options()
def finalize_manifest_options(self):
project = self.distribution
self.ensure_string('abi', self.get_abi())
self.ensure_string('arch', self.get_arch())
self.ensure_categories(project)
self.ensure_string('comment', project.get_description())
self.ensure_desc(project)
self.ensure_string_list('groups')
self.ensure_string('license', self.resolve_license(project))
self.ensure_string('maintainer', self.get_maintainer(project))
self.ensure_name(project)
self.ensure_string('origin', self.get_default_origin(project))
self.ensure_prefix('/usr/local')
self.ensure_string_list('provides')
self.ensure_string_list('requires')
self.ensure_scripts()
self.ensure_string('version', project.get_version())
self.ensure_string_list('users')
self.ensure_string('www', project.get_url())
self.ensure_options()
self.ensure_deps()
self.maybe_rename_console_scripts(project)
def run(self):
self.build_and_install()
self.make_pkg(self.generate_manifest_content())
self.maybe_remove_temp(self.bdist_base)
def build_and_install(self):
if self.use_wheel:
self.build_and_install_via_wheel()
else:
self.build_and_install_via_setuptools()
def build_and_install_via_setuptools(self):
# Basically, we need the intermediate results of bdist_dumb,
# but since it's too monolithic and does the stuff that we would like
# to avoid, here short copy-paste happens /:
build = self.reinitialize_command('build', reinit_subcommands=1)
build.build_base = self.bdist_base
self.run_command('build')
install = self.reinitialize_command('install', reinit_subcommands=1)
install.prefix = self.prefix
install.root = self.install_dir
install.warn_dir = 0
self.run_command('install')
def build_and_install_via_wheel(self):
if not wheel_available:
raise RuntimeError('The `wheel` package is not available.')
build = self.reinitialize_command('build', reinit_subcommands=1)
build.build_base = self.bdist_base
bdist_wheel = self.reinitialize_command(
'bdist_wheel',
reinit_subcommands=1
)
bdist_wheel.bdist_base = self.bdist_base
bdist_wheel.keep_temp = True
self.run_command('bdist_wheel')
name = self.distribution.get_name()
pip.wheel.move_wheel_files(
name=self.name,
req=WhlRequirement.parse('{}=={}'.format(name, self.version)),
wheeldir=bdist_wheel.bdist_dir,
root=self.install_dir,
prefix=self.prefix,
)
def generate_manifest_content(self):
manifest = {
'abi': self.abi,
'arch': self.arch,
'categories': self.categories,
'comment': self.comment,
'deps': self.deps,
'desc': self.desc,
'directories': {},
'files': {},
'flatsize': 0,
'groups': self.groups,
'licenselogic': 'single',
'licenses': [self.license] if self.license else [],
'maintainer': self.maintainer,
'name': self.name,
'options': self.options,
'origin': self.origin,
'prefix': self.prefix,
'provides': self.provides,
'requires': self.requires,
'scripts': self.scripts,
'users': self.users,
'version': self.version,
'www': self.www,
}
mdirs = manifest['directories']
mfiles = manifest['files']
for real_file_path, install_path in self.iter_install_files():
with open(real_file_path, 'rb') as fh:
data = fh.read()
manifest['flatsize'] += len(data)
mdirs[os.path.dirname(install_path)] = {
'gname': 'wheel',
'perm': '0755',
'uname': 'root',
}
mfiles[install_path] = {
'gname': 'wheel',
'perm': '0644',
'sum': hashlib.sha256(data).hexdigest(),
'uname': 'root',
}
# TODO: Should we keep UNKNOWN values?
manifest = {key: value for key, value in manifest.items()
if value and value != 'UNKNOWN'}
if 'name' not in manifest:
raise DistutilsOptionError('Project must have name defined')
if 'version' not in manifest:
raise DistutilsOptionError('Project must have version defined')
if 'comment' not in manifest:
raise DistutilsOptionError('Project must have description defined')
if 'desc' not in manifest:
raise DistutilsOptionError('Project must have long_description'
' defined')
if 'maintainer' not in manifest:
raise DistutilsOptionError('Project must have author or maintainer'
' defined')
return manifest
def make_pkg(self, manifest):
manifest_path = self.make_manifest(manifest)
compact_manifest_path = self.make_compact_manifest(manifest)
files_paths = chain([
(manifest_path, os.path.basename(manifest_path)),
(compact_manifest_path, os.path.basename(compact_manifest_path))
], self.iter_install_files())
self.mkpath(self.dist_dir)
tar_path = self.make_tar(files_paths)
ext = self.format
if ext != 'tar':
compressor = self.get_compressor(ext)
if compressor is None:
raise RuntimeError('Format {} is not supported'.format(ext))
self.compress_tar(tar_path, ext, compressor)
os.remove(tar_path)
def make_manifest(self, content):
path = os.path.join(self.bdist_dir, '+MANIFEST')
with open(path, 'w') as fobj:
json.dump(content, fobj, sort_keys=True, indent=4)
return path
def make_compact_manifest(self, content):
path = os.path.join(self.bdist_dir, '+COMPACT_MANIFEST')
compact_content = content.copy()
compact_content.pop('directories')
compact_content.pop('files')
with open(path, 'w') as fobj:
json.dump(compact_content, fobj, sort_keys=True, indent=4)
return path
def make_tar(self, files_paths):
basename = '{}-{}.tar'.format(self.name, self.version)
path = os.path.join(self.dist_dir, basename)
seen = set()
with tarfile.open(path, 'w') as tar:
for file_path, tar_path in files_paths:
tar_dir_path = os.path.dirname(tar_path)
if tar_dir_path and tar_dir_path not in seen:
tarinfo = tar.gettarinfo(os.path.dirname(file_path),
tar_dir_path)
tarinfo.name = tar_dir_path
tar.addfile(tarinfo)
seen.add(tar_dir_path)
tarinfo = tar.gettarinfo(file_path, tar_path)
tarinfo.name = tar_path
with open(file_path, 'rb') as f:
tar.addfile(tarinfo, f)
return path
def compress_tar(self, tar_path, ext, compressor):
txx_path = tar_path.rsplit('.tar', 1)[0] + '.' + ext
with compressor.open(txx_path, 'w') as txx:
with open(tar_path, 'rb') as tar:
txx.write(tar.read())
return txx_path
def get_compressor(self, format):
return self.compressor_for_format.get(format)
def get_abi(self):
if platform.system().lower() != 'freebsd':
if not self.distribution.is_pure():
raise DistutilsOptionError(
'Unable to determine default ABI value'
' since bdist_pkg call happens not on FreeBSD system.'
' Please specify this value according the target system'
' for which you build this package.'
)
return '*'
return ':'.join((
platform.system(),
# 10.1-STABLE-r273058 -> 10
platform.release().split('-', 1)[0].split('.')[0],
# TODO: ensure that platform.machine() gives correct values
platform.machine()
))
def get_arch(self):
if platform.system().lower() != 'freebsd':
if not self.distribution.is_pure():
raise DistutilsOptionError(
'Unable to determine default ARCH value'
' since bdist_pkg call happens not on FreeBSD system.'
' Please specify this value according the target system'
' for which you build this package.'
)
return '*'
return ':'.join((
platform.system(),
# 10.1-STABLE-r273058 -> 10
platform.release().split('-', 1)[0].split('.')[0],
# TODO: shouldn't there be a better way?
'x86:64' if platform.machine() == 'amd64' else 'x86:32'
))
def get_default_origin(self, project):
return 'devel/py{}{}-{}'.format(sys.version_info[0],
sys.version_info[1],
project.get_name())
def get_maintainer(self, project):
maintainer = '{} <{}>'.format(project.get_maintainer(),
project.get_maintainer_email())
if maintainer == 'UNKNOWN <UNKNOWN>':
# No explicit maintainer specified, use author contact instead
maintainer = '{} <{}>'.format(project.get_author(),
project.get_author_email())
return maintainer
def resolve_license(self, project):
# Thanks for this mapping goes to pytoport project
py2freebsd_mapping = {
'agpl-3.0': 'AGPLv3',
'apache-2.0': 'APACHE20',
'artistic-2.0': 'ART20',
'bsd-2-clause': 'BSD2CLAUSE',
'bsd-3-clause-clear': 'BSD3CLAUSE',
'bsd-3-clause': 'BSD3CLAUSE',
'cc0-1.0': 'CC0-1.0',
'epl-1.0': 'EPL',
'gpl-2.0': 'GPLv2',
'gpl-3.0': 'GPLv3',
'isc': 'ISCL',
'lgpl-2.1': 'LGPL21',
'lgpl-3.0': 'LGPL3',
'mit': 'MIT',
'mpl-2.0': 'MPL',
'ofl-1.1': 'OFL11',
}
license = project.get_license()
pkg_license = py2freebsd_mapping.get(license.lower())
if license != 'UNKNOWN' and pkg_license is None:
self.warn('Unable to convert license %s to PKG naming' % license)
return license
return pkg_license
def ensure_format(self, default):
self.ensure_string('format', default)
if self.format not in {'txz', 'tbz', 'tgz', 'tar'}:
self.warn('Unknown format {!r}, falling back to {}'
''.format(self.format, default))
self.format = default
def ensure_prefix(self, default=None):
self.ensure_string('prefix', default)
self.prefix = self.prefix.rstrip('/')
def ensure_categories(self, project):
self.categories = self.categories or project.get_keywords()
self.ensure_string_list('categories')
def ensure_deps(self):
install_requires = set(self.distribution.install_requires or [])
for option in self.selected_options:
install_requires |= set(self.distribution.extras_require[option])
mapping = self.requirements_mapping or {}
self.deps = self.deps or {}
seen_deps = set([])
for python_dep, spec in mapping.items():
if not isinstance(python_dep, str):
raise DistutilsOptionError('Invalid Python dependency: {}'
''.format(python_dep))
if python_dep not in install_requires:
raise DistutilsOptionError('{} is not in install requires list'
''.format(python_dep))
if not isinstance(spec, dict):
raise DistutilsOptionError('requirements_mapping items must be'
' dict, got {}'.format(repr(spec)))
if set(spec) != {'origin', 'version', 'name'}:
raise DistutilsOptionError('requirements_mapping items must'
' have "origin" and "version" keys,'
' got {}'.format(set(spec)))
for key in {'origin', 'version', 'name'}:
if not isinstance(spec[key], str):
raise DistutilsOptionError('"{}" value must be string, got'
' {}'.format(key, spec[key]))
self.deps[spec['name']] = {'origin': spec['origin'],
'version': spec['version']}
seen_deps.add(python_dep)
missing = seen_deps ^ install_requires
if missing and self.use_pypi_deps:
for item in missing:
requirement = Requirement.parse(item)
distribution = self.package_index.obtain(requirement)
key = 'py{1}{2}-{0}'.format(distribution.key,
*sys.version_info[:2])
self.deps[key] = {
'origin': 'pypi/py-{}'.format(distribution.key),
'version': distribution.version
}
elif missing:
raise DistutilsOptionError('These packages are listed in install'
' requirements, but not in bdist_pkg'
' requirements mapping: {}'
''.format(', '.join(missing)))
def ensure_desc(self, project):
desc = project.get_long_description()
desc = desc if desc != 'UKNOWN' else project.get_description()
desc = self.cut_changelog(desc)
self.ensure_string('desc', desc)
def ensure_name(self, project):
name = project.get_name()
if self.with_py_prefix:
name = 'py{}{}-{}'.format(
sys.version_info[0], sys.version_info[1], name
)
self.ensure_string('name', name)
def ensure_options(self):
provided_options = set(self.distribution.extras_require or {})
self.selected_options = set(self.selected_options or [])
unknown_options = self.selected_options - provided_options
if not unknown_options:
self.options = {option: option in self.selected_options
for option in provided_options}
else:
raise DistutilsOptionError('Unknown extras selected: {}'
''.format(', '.join(unknown_options)))
def ensure_scripts(self):
if self.scripts is None:
return
if not isinstance(self.scripts, dict):
raise DistutilsOptionError('scripts must be a dict, got {}'
''.format(self.scripts))
valid_keys = {
'pre-install',
'post-install',
'install',
'pre-deinstall',
'post-deinstall',
'deinstall',
'pre-upgrade',
'post-upgrade',
'upgrade',
}
bad_keys = [key for key in self.scripts if key not in valid_keys]
if bad_keys:
raise DistutilsOptionError('invalid scripts: {}'
''.format(', '.join(bad_keys)))
bad_keys = [key for key, value in self.scripts.items()
if not isinstance(value, str)]
if bad_keys:
raise DistutilsOptionError('invalid scripts: {}'
''.format(', '.join(bad_keys)))
def iter_install_files(self):
for root, dirs, files in os.walk(self.install_dir):
for file in files:
reldir = os.path.relpath(root, self.install_dir)
install_path = '/' + os.path.join(reldir, file)
install_path = install_path.replace(self.prefix + '/lib64/',
self.prefix + '/lib/')
yield os.path.join(root, file), install_path
def maybe_remove_temp(self, path):
if self.keep_temp:
return
if path is None:
return
if os.path.exists(path):
shutil.rmtree(path)
def maybe_rename_console_scripts(self, project):
if not self.with_py_prefix:
return
if not project.entry_points:
return
console_scripts = project.entry_points.get('console_scripts')
if console_scripts is None:
return
prefixed_console_scripts = []
for script in console_scripts:
name, callback = script.split('=')
name = '{}{}.{}'.format(name.strip(), *sys.version_info[:2])
prefixed_console_scripts.append(
'{} = {}'.format(name, callback.strip())
)
project.entry_points['console_scripts'] = prefixed_console_scripts
def cut_changelog(self, desc):
def match_changelog_header(line):
words = re.findall(r'\b\w+\b', line.lower())
if len(words) != 1:
return True
if 'changelog' in words or 'changes' in words:
return False
return True
return '\n'.join(takewhile(
match_changelog_header,
desc.splitlines()
))
class bdist_pkg(Command):
description = 'create FreeBSD pkg distribution'
user_options = [
('bdist-base=', 'b',
'Base directory for creating built distributions.'),
('dist-dir=', 'd', 'Directory to put distribute files in.'),
('format=', 'f',
'Set format as the package output format. It can be one'
' of txz, tbz, tgz or tar. If an invalid or no format is specified'
' tgz is assumed.'),
('keep-temp', None, 'Keep intermediate build directories and files.'),
('origin=', None, 'Custom origin name for build package.'),
('use-pypi-deps', None,
'Automatically convert unknown Python dependencies to package ones.'
' Note that those dependencies will be named with py{}{}- prefix and'
' assumes that you have such packages in repository.'
''.format(*sys.version_info[:2])),
('use-wheel', None,
'Use bdist_wheel to generated install layout instead of install'
' command.'),
('with-py-prefix', None, 'Prepends py{}{}- prefix to package name.'
''.format(*sys.version_info[:2])),
]
boolean_options = ('keep-temp', 'use-wheel', 'python-deps-to-pkg',
'with-py-prefix')
compressor_for_format = {
'txz': lzma,
'tgz': gzip,
'tbz': bz2,
}
def initialize_options(self):
self.bdist_base = None
self.dist_dir = None
self.format = None
self.keep_temp = False
self.name_prefix = None
self.package_index = PackageIndex()
self.requirements_mapping = None
self.selected_options = None
self.use_pypi_deps = False
self.use_wheel = False
self.with_py_prefix = False
self.initialize_manifest_options()
def initialize_manifest_options(self):
# TODO: What is it and how to use it?
# self.annotations = None
self.abi = None
self.arch = None
self.categories = None
# TODO: Could conflicts be useful for us?
# self.conflicts = None
self.comment = None
# TODO: What is it and how to use it?
# self.dep_formula = None
self.deps = None
self.desc = None
# These fields are autogenerated:
# self.directories = None
# self.dirs = None
# self.files = None
# self.flatsize = None
self.groups = None
self.license = None
self.maintainer = None
# TODO: should that be single message or multiple ones?
# self.messages = None
self.name = None
self.options = None
self.selected_options = None
# Since we use extras, which don't have either defaults or descriptions
# these fields are not supported so far:
# self.options_defaults = None
# self.options_descriptions = None
self.origin = None
# TODO: What is the path?
# self.path = None
self.prefix = None
self.provides = None
self.requires = None
self.scripts = None
# TODO: Do we need shared libs support?
# self.shlibs = None
# self.shlibs_provides = None
# self.shlibs_requires = None
# TODO: Support checksum.
# self.sum = None
self.users = None
self.version = None
# TODO: Can Python packages be vital?
# self.vital = None
self.www = None
def finalize_options(self):
self.set_undefined_options('bdist', ('bdist_base', 'bdist_base'))
self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))
self.ensure_format('tgz')
self.bdist_dir = os.path.join(self.bdist_base, 'pkg')
self.install_dir = os.path.join(self.bdist_dir, 'root')
self.finalize_manifest_options()
def finalize_manifest_options(self):
project = self.distribution
self.ensure_string('abi', self.get_abi())
self.ensure_string('arch', self.get_arch())
self.ensure_categories(project)
self.ensure_string('comment', project.get_description())
self.ensure_desc(project)
self.ensure_string_list('groups')
self.ensure_string('license', self.resolve_license(project))
self.ensure_string('maintainer', self.get_maintainer(project))
self.ensure_name(project)
self.ensure_string('origin', self.get_default_origin(project))
self.ensure_prefix('/usr/local')
self.ensure_string_list('provides')
self.ensure_string_list('requires')
self.ensure_scripts()
self.ensure_string('version', project.get_version())
self.ensure_string_list('users')
self.ensure_string('www', project.get_url())
self.ensure_options()
self.ensure_deps()
self.maybe_rename_console_scripts(project)
def run(self):
self.build_and_install()
self.make_pkg(self.generate_manifest_content())
self.maybe_remove_temp(self.bdist_base)
def build_and_install(self):
if self.use_wheel:
self.build_and_install_via_wheel()
else:
self.build_and_install_via_setuptools()
def build_and_install_via_setuptools(self):
# Basically, we need the intermediate results of bdist_dumb,
# but since it's too monolithic and does the stuff that we would like
# to avoid, here short copy-paste happens /:
build = self.reinitialize_command('build', reinit_subcommands=1)
build.build_base = self.bdist_base
self.run_command('build')
install = self.reinitialize_command('install', reinit_subcommands=1)
install.prefix = self.prefix
install.root = self.install_dir
install.warn_dir = 0
self.run_command('install')
def build_and_install_via_wheel(self):
if not wheel_available:
raise RuntimeError('The `wheel` package is not available.')
build = self.reinitialize_command('build', reinit_subcommands=1)
build.build_base = self.bdist_base
bdist_wheel = self.reinitialize_command('bdist_wheel',
reinit_subcommands=1)
bdist_wheel.bdist_base = self.bdist_base
bdist_wheel.keep_temp = True
self.run_command('bdist_wheel')
name = self.distribution.get_name()
pip.wheel.move_wheel_files(
name=self.name,
req=WhlRequirement.parse('{}=={}'.format(name, self.version)),
wheeldir=bdist_wheel.bdist_dir,
root=self.install_dir,
prefix=self.prefix,
)
def generate_manifest_content(self):
manifest = {
'abi': self.abi,
'arch': self.arch,
'categories': self.categories,
'comment': self.comment,
'deps': self.deps,
'desc': self.desc,
'directories': {},
'files': {},
'flatsize': 0,
'groups': self.groups,
'licenselogic': 'single',
'licenses': [self.license] if self.license else [],
'maintainer': self.maintainer,
'name': self.name,
'options': self.options,
'origin': self.origin,
'prefix': self.prefix,
'provides': self.provides,
'requires': self.requires,
'scripts': self.scripts,
'users': self.users,
'version': self.version,
'www': self.www,
}
mdirs = manifest['directories']
mfiles = manifest['files']
for real_file_path, install_path in self.iter_install_files():
with open(real_file_path, 'rb') as fh:
data = fh.read()
manifest['flatsize'] += len(data)
mdirs[os.path.dirname(install_path)] = {
'gname': 'wheel',
'perm': '0755',
'uname': 'root',
}
mfiles[install_path] = {
'gname': 'wheel',
'perm': '0644',
'sum': hashlib.sha256(data).hexdigest(),
'uname': 'root',
}
# TODO: Should we keep UNKNOWN values?
manifest = {
key: value
for key, value in manifest.items() if value and value != 'UNKNOWN'
}
if 'name' not in manifest:
raise DistutilsOptionError('Project must have name defined')
if 'version' not in manifest:
raise DistutilsOptionError('Project must have version defined')
if 'comment' not in manifest:
raise DistutilsOptionError('Project must have description defined')
if 'desc' not in manifest:
raise DistutilsOptionError('Project must have long_description'
' defined')
if 'maintainer' not in manifest:
raise DistutilsOptionError('Project must have author or maintainer'
' defined')
return manifest
def make_pkg(self, manifest):
manifest_path = self.make_manifest(manifest)
compact_manifest_path = self.make_compact_manifest(manifest)
files_paths = chain(
[(manifest_path, os.path.basename(manifest_path)),
(compact_manifest_path, os.path.basename(compact_manifest_path))],
self.iter_install_files())
self.mkpath(self.dist_dir)
tar_path = self.make_tar(files_paths)
ext = self.format
if ext != 'tar':
compressor = self.get_compressor(ext)
if compressor is None:
raise RuntimeError('Format {} is not supported'.format(ext))
self.compress_tar(tar_path, ext, compressor)
os.remove(tar_path)
def make_manifest(self, content):
path = os.path.join(self.bdist_dir, '+MANIFEST')
with open(path, 'w') as fobj:
json.dump(content, fobj, sort_keys=True, indent=4)
return path
def make_compact_manifest(self, content):
path = os.path.join(self.bdist_dir, '+COMPACT_MANIFEST')
compact_content = content.copy()
compact_content.pop('directories')
compact_content.pop('files')
with open(path, 'w') as fobj:
json.dump(compact_content, fobj, sort_keys=True, indent=4)
return path
def make_tar(self, files_paths):
basename = '{}-{}.tar'.format(self.name, self.version)
path = os.path.join(self.dist_dir, basename)
seen = set()
with tarfile.open(path, 'w') as tar:
for file_path, tar_path in files_paths:
tar_dir_path = os.path.dirname(tar_path)
if tar_dir_path and tar_dir_path not in seen:
tarinfo = tar.gettarinfo(os.path.dirname(file_path),
tar_dir_path)
tarinfo.name = tar_dir_path
tar.addfile(tarinfo)
seen.add(tar_dir_path)
tarinfo = tar.gettarinfo(file_path, tar_path)
tarinfo.name = tar_path
with open(file_path, 'rb') as f:
tar.addfile(tarinfo, f)
return path
def compress_tar(self, tar_path, ext, compressor):
txx_path = tar_path.rsplit('.tar', 1)[0] + '.' + ext
with compressor.open(txx_path, 'w') as txx:
with open(tar_path, 'rb') as tar:
txx.write(tar.read())
return txx_path
def get_compressor(self, format):
return self.compressor_for_format.get(format)
def get_abi(self):
if platform.system().lower() != 'freebsd':
if not self.distribution.is_pure():
raise DistutilsOptionError(
'Unable to determine default ABI value'
' since bdist_pkg call happens not on FreeBSD system.'
' Please specify this value according the target system'
' for which you build this package.')
return '*'
return ':'.join((
platform.system(),
# 10.1-STABLE-r273058 -> 10
platform.release().split('-', 1)[0].split('.')[0],
# TODO: ensure that platform.machine() gives correct values
platform.machine()))
def get_arch(self):
if platform.system().lower() != 'freebsd':
if not self.distribution.is_pure():
raise DistutilsOptionError(
'Unable to determine default ARCH value'
' since bdist_pkg call happens not on FreeBSD system.'
' Please specify this value according the target system'
' for which you build this package.')
return '*'
return ':'.join((
platform.system(),
# 10.1-STABLE-r273058 -> 10
platform.release().split('-', 1)[0].split('.')[0],
# TODO: shouldn't there be a better way?
'x86:64' if platform.machine() == 'amd64' else 'x86:32'))
def get_default_origin(self, project):
return 'devel/py{}{}-{}'.format(sys.version_info[0],
sys.version_info[1],
project.get_name())
def get_maintainer(self, project):
maintainer = '{} <{}>'.format(project.get_maintainer(),
project.get_maintainer_email())
if maintainer == 'UNKNOWN <UNKNOWN>':
# No explicit maintainer specified, use author contact instead
maintainer = '{} <{}>'.format(project.get_author(),
project.get_author_email())
return maintainer
def resolve_license(self, project):
# Thanks for this mapping goes to pytoport project
py2freebsd_mapping = {
'agpl-3.0': 'AGPLv3',
'apache-2.0': 'APACHE20',
'artistic-2.0': 'ART20',
'bsd-2-clause': 'BSD2CLAUSE',
'bsd-3-clause-clear': 'BSD3CLAUSE',
'bsd-3-clause': 'BSD3CLAUSE',
'cc0-1.0': 'CC0-1.0',
'epl-1.0': 'EPL',
'gpl-2.0': 'GPLv2',
'gpl-3.0': 'GPLv3',
'isc': 'ISCL',
'lgpl-2.1': 'LGPL21',
'lgpl-3.0': 'LGPL3',
'mit': 'MIT',
'mpl-2.0': 'MPL',
'ofl-1.1': 'OFL11',
}
license = project.get_license()
pkg_license = py2freebsd_mapping.get(license.lower())
if license != 'UNKNOWN' and pkg_license is None:
self.warn('Unable to convert license %s to PKG naming' % license)
return license
return pkg_license
def ensure_format(self, default):
self.ensure_string('format', default)
if self.format not in {'txz', 'tbz', 'tgz', 'tar'}:
self.warn('Unknown format {!r}, falling back to {}'
''.format(self.format, default))
self.format = default
def ensure_prefix(self, default=None):
self.ensure_string('prefix', default)
self.prefix = self.prefix.rstrip('/')
def ensure_categories(self, project):
self.categories = self.categories or project.get_keywords()
self.ensure_string_list('categories')
def ensure_deps(self):
install_requires = set(self.distribution.install_requires or [])
for option in self.selected_options:
install_requires |= set(self.distribution.extras_require[option])
mapping = self.requirements_mapping or {}
self.deps = self.deps or {}
seen_deps = set([])
for python_dep, spec in mapping.items():
if not isinstance(python_dep, str):
raise DistutilsOptionError('Invalid Python dependency: {}'
''.format(python_dep))
if python_dep not in install_requires:
raise DistutilsOptionError('{} is not in install requires list'
''.format(python_dep))
if not isinstance(spec, dict):
raise DistutilsOptionError('requirements_mapping items must be'
' dict, got {}'.format(repr(spec)))
if set(spec) != {'origin', 'version', 'name'}:
raise DistutilsOptionError('requirements_mapping items must'
' have "origin" and "version" keys,'
' got {}'.format(set(spec)))
for key in {'origin', 'version', 'name'}:
if not isinstance(spec[key], str):
raise DistutilsOptionError('"{}" value must be string, got'
' {}'.format(key, spec[key]))
self.deps[spec['name']] = {
'origin': spec['origin'],
'version': spec['version']
}
seen_deps.add(python_dep)
missing = seen_deps ^ install_requires
if missing and self.use_pypi_deps:
for item in missing:
requirement = Requirement.parse(item)
distribution = self.package_index.obtain(requirement)
key = 'py{1}{2}-{0}'.format(distribution.key,
*sys.version_info[:2])
self.deps[key] = {
'origin': 'pypi/py-{}'.format(distribution.key),
'version': distribution.version
}
elif missing:
raise DistutilsOptionError('These packages are listed in install'
' requirements, but not in bdist_pkg'
' requirements mapping: {}'
''.format(', '.join(missing)))
def ensure_desc(self, project):
desc = project.get_long_description()
desc = desc if desc != 'UKNOWN' else project.get_description()
desc = self.cut_changelog(desc)
self.ensure_string('desc', desc)
def ensure_name(self, project):
name = project.get_name()
if self.with_py_prefix:
name = 'py{}{}-{}'.format(sys.version_info[0], sys.version_info[1],
name)
self.ensure_string('name', name)
def ensure_options(self):
provided_options = set(self.distribution.extras_require or {})
self.selected_options = set(self.selected_options or [])
unknown_options = self.selected_options - provided_options
if not unknown_options:
self.options = {
option: option in self.selected_options
for option in provided_options
}
else:
raise DistutilsOptionError('Unknown extras selected: {}'
''.format(', '.join(unknown_options)))
def ensure_scripts(self):
if self.scripts is None:
return
if not isinstance(self.scripts, dict):
raise DistutilsOptionError('scripts must be a dict, got {}'
''.format(self.scripts))
valid_keys = {
'pre-install',
'post-install',
'install',
'pre-deinstall',
'post-deinstall',
'deinstall',
'pre-upgrade',
'post-upgrade',
'upgrade',
}
bad_keys = [key for key in self.scripts if key not in valid_keys]
if bad_keys:
raise DistutilsOptionError('invalid scripts: {}'
''.format(', '.join(bad_keys)))
bad_keys = [
key for key, value in self.scripts.items()
if not isinstance(value, str)
]
if bad_keys:
raise DistutilsOptionError('invalid scripts: {}'
''.format(', '.join(bad_keys)))
def iter_install_files(self):
for root, dirs, files in os.walk(self.install_dir):
for file in files:
reldir = os.path.relpath(root, self.install_dir)
install_path = '/' + os.path.join(reldir, file)
install_path = install_path.replace(self.prefix + '/lib64/',
self.prefix + '/lib/')
yield os.path.join(root, file), install_path
def maybe_remove_temp(self, path):
if self.keep_temp:
return
if path is None:
return
if os.path.exists(path):
shutil.rmtree(path)
def maybe_rename_console_scripts(self, project):
if not self.with_py_prefix:
return
if not project.entry_points:
return
console_scripts = project.entry_points.get('console_scripts')
if console_scripts is None:
return
prefixed_console_scripts = []
for script in console_scripts:
name, callback = script.split('=')
name = '{}{}.{}'.format(name.strip(), *sys.version_info[:2])
prefixed_console_scripts.append('{} = {}'.format(
name, callback.strip()))
project.entry_points['console_scripts'] = prefixed_console_scripts
def cut_changelog(self, desc):
def match_changelog_header(line):
words = re.findall(r'\b\w+\b', line.lower())
if len(words) != 1:
return True
if 'changelog' in words or 'changes' in words:
return False
return True
return '\n'.join(takewhile(match_changelog_header, desc.splitlines()))
def runit():
# process command-line options
bool_opts = map(translate_longopt, stdeb_cmd_bool_opts)
bool_opts.append("process-dependencies")
parser = FancyGetopt(stdeb_cmdline_opts + [("help", "h", "show detailed help message")] + EXTRA_OPTS)
optobj = OptObj()
args = parser.getopt(object=optobj)
idx = PackageIndex()
for option in optobj.__dict__:
value = getattr(optobj, option)
is_string = type(value) == str
if option in bool_opts and is_string:
setattr(optobj, option, strtobool(value))
if hasattr(optobj, "help"):
print USAGE
parser.set_option_table(stdeb_cmdline_opts + EXTRA_OPTS)
parser.print_help("Options:")
return 0
if len(args) != 1:
log.error("not given single argument (distfile), args=%r", args)
print USAGE
return 1
sdist_file = args[0]
package = None
final_dist_dir = optobj.__dict__.get("dist_dir", "deb_dist")
tmp_dist_dir = os.path.join(final_dist_dir, "tmp_py2dsc")
if os.path.exists(tmp_dist_dir):
shutil.rmtree(tmp_dist_dir)
os.makedirs(tmp_dist_dir)
if not os.path.isfile(sdist_file):
for ext in EXTENSIONS:
if sdist_file.endswith(ext):
raise IOError, "File not found"
package = Requirement.parse(sdist_file)
log.info("Package %s not found, trying PyPI..." % sdist_file)
dist = idx.fetch_distribution(package, final_dist_dir, force_scan=True, source=True)
if hasattr(dist, "location"):
sdist_file = dist.location
else:
raise Exception, "Distribution not found on PyPi"
log.info("Got %s", sdist_file)
dist = list(distros_for_filename(sdist_file))[0]
idx.scan_egg_links(dist.location)
package = idx.obtain(Requirement.parse(dist.project_name))
if hasattr(optobj, "process_dependencies"):
if bool(int(getattr(optobj, "process_dependencies"))):
backup_argv = sys.argv[:]
oldargv = sys.argv[:]
oldargv.pop(-1)
if package.requires():
log.info("Processing package dependencies for %s", package)
for req in package.requires():
# print >> sys.stderr
new_argv = oldargv + ["%s" % req]
log.info("Bulding dependency package %s", req)
log.info(" running '%s'", " ".join(new_argv))
sys.argv = new_argv
runit()
# print >> sys.stderr
if package.requires():
log.info("Completed building dependencies " "for %s, continuing...", package)
sys.argv = backup_argv
if package is not None and hasattr(optobj, "extra_cfg_file"):
# Allow one to have patch-files setup on config file for example
local_parser = SafeConfigParser()
local_parser.readfp(open(optobj.__dict__.get("extra_cfg_file")))
if local_parser.has_section(package.project_name):
for opt in local_parser.options(package.project_name):
_opt = opt.replace("_", "-")
if parser.has_option(_opt) or parser.has_option(_opt + "="):
setattr(optobj, opt, local_parser.get(package.project_name, opt))
patch_file = optobj.__dict__.get("patch_file", None)
patch_level = int(optobj.__dict__.get("patch_level", 0))
patch_posix = int(optobj.__dict__.get("patch_posix", 0))
expand_dir = os.path.join(tmp_dist_dir, "stdeb_tmp")
if os.path.exists(expand_dir):
shutil.rmtree(expand_dir)
if not os.path.exists(tmp_dist_dir):
os.mkdir(tmp_dist_dir)
os.mkdir(expand_dir)
expand_sdist_file(os.path.abspath(sdist_file), cwd=expand_dir)
# now the sdist package is expanded in expand_dir
expanded_root_files = os.listdir(expand_dir)
assert len(expanded_root_files) == 1
repackaged_dirname = expanded_root_files[0]
fullpath_repackaged_dirname = os.path.join(tmp_dist_dir, repackaged_dirname)
base_dir = os.path.join(expand_dir, expanded_root_files[0])
if os.path.exists(fullpath_repackaged_dirname):
# prevent weird build errors if this dir exists
shutil.rmtree(fullpath_repackaged_dirname)
os.renames(base_dir, fullpath_repackaged_dirname)
del base_dir # no longer useful
##############################################
if patch_file is not None:
log.info("py2dsc applying patch %s", patch_file)
apply_patch(patch_file, posix=patch_posix, level=patch_level, cwd=fullpath_repackaged_dirname)
patch_already_applied = 1
else:
patch_already_applied = 0
##############################################
abs_dist_dir = os.path.abspath(final_dist_dir)
extra_args = []
for long in parser.long_opts:
if long in ["dist-dir=", "patch-file=", "process-dependencies"]:
continue # dealt with by this invocation
attr = parser.get_attr_name(long).rstrip("=")
if hasattr(optobj, attr):
val = getattr(optobj, attr)
if attr == "extra_cfg_file":
val = os.path.abspath(val)
if long in bool_opts or long.replace("-", "_") in bool_opts:
extra_args.append("--%s" % long)
else:
extra_args.append("--" + long + str(val))
if patch_already_applied == 1:
extra_args.append("--patch-already-applied")
args = [
sys.executable,
"-c",
"import stdeb, sys; f='setup.py'; " + "sys.argv[0]=f; execfile(f,{'__file__':f,'__name__':'__main__'})",
"sdist_dsc",
"--dist-dir=%s" % abs_dist_dir,
"--use-premade-distfile=%s" % os.path.abspath(sdist_file),
] + extra_args
log.info("-=" * 35 + "-")
# print >> sys.stderr, '-='*20
# print >> sys.stderr, "Note that the .cfg file(s), if present, have not "\
# "been read at this stage. If options are necessary, pass them from "\
# "the command line"
log.info("running the following command in directory: %s\n%s", fullpath_repackaged_dirname, " ".join(args))
log.info("-=" * 35 + "-")
try:
returncode = subprocess.call(args, cwd=fullpath_repackaged_dirname)
except:
log.error("ERROR running: %s", " ".join(args))
log.error("ERROR in %s", fullpath_repackaged_dirname)
raise
if returncode:
log.error("ERROR running: %s", " ".join(args))
log.error("ERROR in %s", fullpath_repackaged_dirname)
# log.error(' stderr: %s'res.stderr.read())
# print >> sys.stderr, 'ERROR running: %s'%(' '.join(args),)
# print >> sys.stderr, res.stderr.read()
return returncode
# raise RuntimeError('returncode %d'%returncode)
# result = res.stdout.read().strip()
shutil.rmtree(tmp_dist_dir)
return returncode
