def _do_upgrade(self, dist): # Build up a requirement for a higher bugfix release but a lower minor # release (so API compatibility is guaranteed) next_version = _next_version(dist.parsed_version) req = pkg_resources.Requirement.parse( '{0}>{1},<{2}'.format(DIST_NAME, dist.version, next_version)) package_index = PackageIndex(index_url=self.index_url) upgrade = package_index.obtain(req) if upgrade is not None: return self._do_download(version=upgrade.version)
def _do_upgrade(self, dist): # Build up a requirement for a higher bugfix release but a lower minor # release (so API compatibility is guaranteed) next_version = _next_version(dist.parsed_version) req = pkg_resources.Requirement.parse('{0}>{1},<{2}'.format( DIST_NAME, dist.version, next_version)) package_index = PackageIndex(index_url=self.index_url) upgrade = package_index.obtain(req) if upgrade is not None: return self._do_download(version=upgrade.version)
def _do_upgrade(dist, index_url): # Build up a requirement for a higher bugfix release but a lower minor # release (so API compatibility is guaranteed) # sketchy version parsing--maybe come up with something a bit more # robust for this major, minor = (int(part) for part in dist.parsed_version[:2]) next_minor = '.'.join([str(major), str(minor + 1), '0']) req = pkg_resources.Requirement.parse( '{0}>{1},<{2}'.format(DIST_NAME, dist.version, next_minor)) package_index = PackageIndex(index_url=index_url) upgrade = package_index.obtain(req) if upgrade is not None: return _do_download(version=upgrade.version, index_url=index_url)
class bdist_pkg(Command): description = 'create FreeBSD pkg distribution' user_options = [ ('bdist-base=', 'b', 'Base directory for creating built distributions.'), ('dist-dir=', 'd', 'Directory to put distribute files in.'), ('format=', 'f', 'Set format as the package output format. It can be one' ' of txz, tbz, tgz or tar. If an invalid or no format is specified' ' tgz is assumed.'), ('keep-temp', None, 'Keep intermediate build directories and files.'), ('origin=', None, 'Custom origin name for build package.'), ('use-pypi-deps', None, 'Automatically convert unknown Python dependencies to package ones.' ' Note that those dependencies will be named with py{}{}- prefix and' ' assumes that you have such packages in repository.' ''.format(*sys.version_info[:2])), ('use-wheel', None, 'Use bdist_wheel to generated install layout instead of install' ' command.'), ('with-py-prefix', None, 'Prepends py{}{}- prefix to package name.' ''.format(*sys.version_info[:2])), ] boolean_options = ('keep-temp', 'use-wheel', 'python-deps-to-pkg', 'with-py-prefix') compressor_for_format = { 'txz': lzma, 'tgz': gzip, 'tbz': bz2, } def initialize_options(self): self.bdist_base = None self.dist_dir = None self.format = None self.keep_temp = False self.name_prefix = None self.package_index = PackageIndex() self.requirements_mapping = None self.selected_options = None self.use_pypi_deps = False self.use_wheel = False self.with_py_prefix = False self.initialize_manifest_options() def initialize_manifest_options(self): # TODO: What is it and how to use it? # self.annotations = None self.abi = None self.arch = None self.categories = None # TODO: Could conflicts be useful for us? # self.conflicts = None self.comment = None # TODO: What is it and how to use it? # self.dep_formula = None self.deps = None self.desc = None # These fields are autogenerated: # self.directories = None # self.dirs = None # self.files = None # self.flatsize = None self.groups = None self.license = None self.maintainer = None # TODO: should that be single message or multiple ones? # self.messages = None self.name = None self.options = None self.selected_options = None # Since we use extras, which don't have either defaults or descriptions # these fields are not supported so far: # self.options_defaults = None # self.options_descriptions = None self.origin = None # TODO: What is the path? # self.path = None self.prefix = None self.provides = None self.requires = None self.scripts = None # TODO: Do we need shared libs support? # self.shlibs = None # self.shlibs_provides = None # self.shlibs_requires = None # TODO: Support checksum. # self.sum = None self.users = None self.version = None # TODO: Can Python packages be vital? # self.vital = None self.www = None def finalize_options(self): self.set_undefined_options('bdist', ('bdist_base', 'bdist_base')) self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) self.ensure_format('tgz') self.bdist_dir = os.path.join(self.bdist_base, 'pkg') self.install_dir = os.path.join(self.bdist_dir, 'root') self.finalize_manifest_options() def finalize_manifest_options(self): project = self.distribution self.ensure_string('abi', self.get_abi()) self.ensure_string('arch', self.get_arch()) self.ensure_categories(project) self.ensure_string('comment', project.get_description()) self.ensure_desc(project) self.ensure_string_list('groups') self.ensure_string('license', self.resolve_license(project)) self.ensure_string('maintainer', self.get_maintainer(project)) self.ensure_name(project) self.ensure_string('origin', self.get_default_origin(project)) self.ensure_prefix('/usr/local') self.ensure_string_list('provides') self.ensure_string_list('requires') self.ensure_scripts() self.ensure_string('version', project.get_version()) self.ensure_string_list('users') self.ensure_string('www', project.get_url()) self.ensure_options() self.ensure_deps() self.maybe_rename_console_scripts(project) def run(self): self.build_and_install() self.make_pkg(self.generate_manifest_content()) self.maybe_remove_temp(self.bdist_base) def build_and_install(self): if self.use_wheel: self.build_and_install_via_wheel() else: self.build_and_install_via_setuptools() def build_and_install_via_setuptools(self): # Basically, we need the intermediate results of bdist_dumb, # but since it's too monolithic and does the stuff that we would like # to avoid, here short copy-paste happens /: build = self.reinitialize_command('build', reinit_subcommands=1) build.build_base = self.bdist_base self.run_command('build') install = self.reinitialize_command('install', reinit_subcommands=1) install.prefix = self.prefix install.root = self.install_dir install.warn_dir = 0 self.run_command('install') def build_and_install_via_wheel(self): if not wheel_available: raise RuntimeError('The `wheel` package is not available.') build = self.reinitialize_command('build', reinit_subcommands=1) build.build_base = self.bdist_base bdist_wheel = self.reinitialize_command( 'bdist_wheel', reinit_subcommands=1 ) bdist_wheel.bdist_base = self.bdist_base bdist_wheel.keep_temp = True self.run_command('bdist_wheel') name = self.distribution.get_name() pip.wheel.move_wheel_files( name=self.name, req=WhlRequirement.parse('{}=={}'.format(name, self.version)), wheeldir=bdist_wheel.bdist_dir, root=self.install_dir, prefix=self.prefix, ) def generate_manifest_content(self): manifest = { 'abi': self.abi, 'arch': self.arch, 'categories': self.categories, 'comment': self.comment, 'deps': self.deps, 'desc': self.desc, 'directories': {}, 'files': {}, 'flatsize': 0, 'groups': self.groups, 'licenselogic': 'single', 'licenses': [self.license] if self.license else [], 'maintainer': self.maintainer, 'name': self.name, 'options': self.options, 'origin': self.origin, 'prefix': self.prefix, 'provides': self.provides, 'requires': self.requires, 'scripts': self.scripts, 'users': self.users, 'version': self.version, 'www': self.www, } mdirs = manifest['directories'] mfiles = manifest['files'] for real_file_path, install_path in self.iter_install_files(): with open(real_file_path, 'rb') as fh: data = fh.read() manifest['flatsize'] += len(data) mdirs[os.path.dirname(install_path)] = { 'gname': 'wheel', 'perm': '0755', 'uname': 'root', } mfiles[install_path] = { 'gname': 'wheel', 'perm': '0644', 'sum': hashlib.sha256(data).hexdigest(), 'uname': 'root', } # TODO: Should we keep UNKNOWN values? manifest = {key: value for key, value in manifest.items() if value and value != 'UNKNOWN'} if 'name' not in manifest: raise DistutilsOptionError('Project must have name defined') if 'version' not in manifest: raise DistutilsOptionError('Project must have version defined') if 'comment' not in manifest: raise DistutilsOptionError('Project must have description defined') if 'desc' not in manifest: raise DistutilsOptionError('Project must have long_description' ' defined') if 'maintainer' not in manifest: raise DistutilsOptionError('Project must have author or maintainer' ' defined') return manifest def make_pkg(self, manifest): manifest_path = self.make_manifest(manifest) compact_manifest_path = self.make_compact_manifest(manifest) files_paths = chain([ (manifest_path, os.path.basename(manifest_path)), (compact_manifest_path, os.path.basename(compact_manifest_path)) ], self.iter_install_files()) self.mkpath(self.dist_dir) tar_path = self.make_tar(files_paths) ext = self.format if ext != 'tar': compressor = self.get_compressor(ext) if compressor is None: raise RuntimeError('Format {} is not supported'.format(ext)) self.compress_tar(tar_path, ext, compressor) os.remove(tar_path) def make_manifest(self, content): path = os.path.join(self.bdist_dir, '+MANIFEST') with open(path, 'w') as fobj: json.dump(content, fobj, sort_keys=True, indent=4) return path def make_compact_manifest(self, content): path = os.path.join(self.bdist_dir, '+COMPACT_MANIFEST') compact_content = content.copy() compact_content.pop('directories') compact_content.pop('files') with open(path, 'w') as fobj: json.dump(compact_content, fobj, sort_keys=True, indent=4) return path def make_tar(self, files_paths): basename = '{}-{}.tar'.format(self.name, self.version) path = os.path.join(self.dist_dir, basename) seen = set() with tarfile.open(path, 'w') as tar: for file_path, tar_path in files_paths: tar_dir_path = os.path.dirname(tar_path) if tar_dir_path and tar_dir_path not in seen: tarinfo = tar.gettarinfo(os.path.dirname(file_path), tar_dir_path) tarinfo.name = tar_dir_path tar.addfile(tarinfo) seen.add(tar_dir_path) tarinfo = tar.gettarinfo(file_path, tar_path) tarinfo.name = tar_path with open(file_path, 'rb') as f: tar.addfile(tarinfo, f) return path def compress_tar(self, tar_path, ext, compressor): txx_path = tar_path.rsplit('.tar', 1)[0] + '.' + ext with compressor.open(txx_path, 'w') as txx: with open(tar_path, 'rb') as tar: txx.write(tar.read()) return txx_path def get_compressor(self, format): return self.compressor_for_format.get(format) def get_abi(self): if platform.system().lower() != 'freebsd': if not self.distribution.is_pure(): raise DistutilsOptionError( 'Unable to determine default ABI value' ' since bdist_pkg call happens not on FreeBSD system.' ' Please specify this value according the target system' ' for which you build this package.' ) return '*' return ':'.join(( platform.system(), # 10.1-STABLE-r273058 -> 10 platform.release().split('-', 1)[0].split('.')[0], # TODO: ensure that platform.machine() gives correct values platform.machine() )) def get_arch(self): if platform.system().lower() != 'freebsd': if not self.distribution.is_pure(): raise DistutilsOptionError( 'Unable to determine default ARCH value' ' since bdist_pkg call happens not on FreeBSD system.' ' Please specify this value according the target system' ' for which you build this package.' ) return '*' return ':'.join(( platform.system(), # 10.1-STABLE-r273058 -> 10 platform.release().split('-', 1)[0].split('.')[0], # TODO: shouldn't there be a better way? 'x86:64' if platform.machine() == 'amd64' else 'x86:32' )) def get_default_origin(self, project): return 'devel/py{}{}-{}'.format(sys.version_info[0], sys.version_info[1], project.get_name()) def get_maintainer(self, project): maintainer = '{} <{}>'.format(project.get_maintainer(), project.get_maintainer_email()) if maintainer == 'UNKNOWN <UNKNOWN>': # No explicit maintainer specified, use author contact instead maintainer = '{} <{}>'.format(project.get_author(), project.get_author_email()) return maintainer def resolve_license(self, project): # Thanks for this mapping goes to pytoport project py2freebsd_mapping = { 'agpl-3.0': 'AGPLv3', 'apache-2.0': 'APACHE20', 'artistic-2.0': 'ART20', 'bsd-2-clause': 'BSD2CLAUSE', 'bsd-3-clause-clear': 'BSD3CLAUSE', 'bsd-3-clause': 'BSD3CLAUSE', 'cc0-1.0': 'CC0-1.0', 'epl-1.0': 'EPL', 'gpl-2.0': 'GPLv2', 'gpl-3.0': 'GPLv3', 'isc': 'ISCL', 'lgpl-2.1': 'LGPL21', 'lgpl-3.0': 'LGPL3', 'mit': 'MIT', 'mpl-2.0': 'MPL', 'ofl-1.1': 'OFL11', } license = project.get_license() pkg_license = py2freebsd_mapping.get(license.lower()) if license != 'UNKNOWN' and pkg_license is None: self.warn('Unable to convert license %s to PKG naming' % license) return license return pkg_license def ensure_format(self, default): self.ensure_string('format', default) if self.format not in {'txz', 'tbz', 'tgz', 'tar'}: self.warn('Unknown format {!r}, falling back to {}' ''.format(self.format, default)) self.format = default def ensure_prefix(self, default=None): self.ensure_string('prefix', default) self.prefix = self.prefix.rstrip('/') def ensure_categories(self, project): self.categories = self.categories or project.get_keywords() self.ensure_string_list('categories') def ensure_deps(self): install_requires = set(self.distribution.install_requires or []) for option in self.selected_options: install_requires |= set(self.distribution.extras_require[option]) mapping = self.requirements_mapping or {} self.deps = self.deps or {} seen_deps = set([]) for python_dep, spec in mapping.items(): if not isinstance(python_dep, str): raise DistutilsOptionError('Invalid Python dependency: {}' ''.format(python_dep)) if python_dep not in install_requires: raise DistutilsOptionError('{} is not in install requires list' ''.format(python_dep)) if not isinstance(spec, dict): raise DistutilsOptionError('requirements_mapping items must be' ' dict, got {}'.format(repr(spec))) if set(spec) != {'origin', 'version', 'name'}: raise DistutilsOptionError('requirements_mapping items must' ' have "origin" and "version" keys,' ' got {}'.format(set(spec))) for key in {'origin', 'version', 'name'}: if not isinstance(spec[key], str): raise DistutilsOptionError('"{}" value must be string, got' ' {}'.format(key, spec[key])) self.deps[spec['name']] = {'origin': spec['origin'], 'version': spec['version']} seen_deps.add(python_dep) missing = seen_deps ^ install_requires if missing and self.use_pypi_deps: for item in missing: requirement = Requirement.parse(item) distribution = self.package_index.obtain(requirement) key = 'py{1}{2}-{0}'.format(distribution.key, *sys.version_info[:2]) self.deps[key] = { 'origin': 'pypi/py-{}'.format(distribution.key), 'version': distribution.version } elif missing: raise DistutilsOptionError('These packages are listed in install' ' requirements, but not in bdist_pkg' ' requirements mapping: {}' ''.format(', '.join(missing))) def ensure_desc(self, project): desc = project.get_long_description() desc = desc if desc != 'UKNOWN' else project.get_description() desc = self.cut_changelog(desc) self.ensure_string('desc', desc) def ensure_name(self, project): name = project.get_name() if self.with_py_prefix: name = 'py{}{}-{}'.format( sys.version_info[0], sys.version_info[1], name ) self.ensure_string('name', name) def ensure_options(self): provided_options = set(self.distribution.extras_require or {}) self.selected_options = set(self.selected_options or []) unknown_options = self.selected_options - provided_options if not unknown_options: self.options = {option: option in self.selected_options for option in provided_options} else: raise DistutilsOptionError('Unknown extras selected: {}' ''.format(', '.join(unknown_options))) def ensure_scripts(self): if self.scripts is None: return if not isinstance(self.scripts, dict): raise DistutilsOptionError('scripts must be a dict, got {}' ''.format(self.scripts)) valid_keys = { 'pre-install', 'post-install', 'install', 'pre-deinstall', 'post-deinstall', 'deinstall', 'pre-upgrade', 'post-upgrade', 'upgrade', } bad_keys = [key for key in self.scripts if key not in valid_keys] if bad_keys: raise DistutilsOptionError('invalid scripts: {}' ''.format(', '.join(bad_keys))) bad_keys = [key for key, value in self.scripts.items() if not isinstance(value, str)] if bad_keys: raise DistutilsOptionError('invalid scripts: {}' ''.format(', '.join(bad_keys))) def iter_install_files(self): for root, dirs, files in os.walk(self.install_dir): for file in files: reldir = os.path.relpath(root, self.install_dir) install_path = '/' + os.path.join(reldir, file) install_path = install_path.replace(self.prefix + '/lib64/', self.prefix + '/lib/') yield os.path.join(root, file), install_path def maybe_remove_temp(self, path): if self.keep_temp: return if path is None: return if os.path.exists(path): shutil.rmtree(path) def maybe_rename_console_scripts(self, project): if not self.with_py_prefix: return if not project.entry_points: return console_scripts = project.entry_points.get('console_scripts') if console_scripts is None: return prefixed_console_scripts = [] for script in console_scripts: name, callback = script.split('=') name = '{}{}.{}'.format(name.strip(), *sys.version_info[:2]) prefixed_console_scripts.append( '{} = {}'.format(name, callback.strip()) ) project.entry_points['console_scripts'] = prefixed_console_scripts def cut_changelog(self, desc): def match_changelog_header(line): words = re.findall(r'\b\w+\b', line.lower()) if len(words) != 1: return True if 'changelog' in words or 'changes' in words: return False return True return '\n'.join(takewhile( match_changelog_header, desc.splitlines() ))
class bdist_pkg(Command): description = 'create FreeBSD pkg distribution' user_options = [ ('bdist-base=', 'b', 'Base directory for creating built distributions.'), ('dist-dir=', 'd', 'Directory to put distribute files in.'), ('format=', 'f', 'Set format as the package output format. It can be one' ' of txz, tbz, tgz or tar. If an invalid or no format is specified' ' tgz is assumed.'), ('keep-temp', None, 'Keep intermediate build directories and files.'), ('origin=', None, 'Custom origin name for build package.'), ('use-pypi-deps', None, 'Automatically convert unknown Python dependencies to package ones.' ' Note that those dependencies will be named with py{}{}- prefix and' ' assumes that you have such packages in repository.' ''.format(*sys.version_info[:2])), ('use-wheel', None, 'Use bdist_wheel to generated install layout instead of install' ' command.'), ('with-py-prefix', None, 'Prepends py{}{}- prefix to package name.' ''.format(*sys.version_info[:2])), ] boolean_options = ('keep-temp', 'use-wheel', 'python-deps-to-pkg', 'with-py-prefix') compressor_for_format = { 'txz': lzma, 'tgz': gzip, 'tbz': bz2, } def initialize_options(self): self.bdist_base = None self.dist_dir = None self.format = None self.keep_temp = False self.name_prefix = None self.package_index = PackageIndex() self.requirements_mapping = None self.selected_options = None self.use_pypi_deps = False self.use_wheel = False self.with_py_prefix = False self.initialize_manifest_options() def initialize_manifest_options(self): # TODO: What is it and how to use it? # self.annotations = None self.abi = None self.arch = None self.categories = None # TODO: Could conflicts be useful for us? # self.conflicts = None self.comment = None # TODO: What is it and how to use it? # self.dep_formula = None self.deps = None self.desc = None # These fields are autogenerated: # self.directories = None # self.dirs = None # self.files = None # self.flatsize = None self.groups = None self.license = None self.maintainer = None # TODO: should that be single message or multiple ones? # self.messages = None self.name = None self.options = None self.selected_options = None # Since we use extras, which don't have either defaults or descriptions # these fields are not supported so far: # self.options_defaults = None # self.options_descriptions = None self.origin = None # TODO: What is the path? # self.path = None self.prefix = None self.provides = None self.requires = None self.scripts = None # TODO: Do we need shared libs support? # self.shlibs = None # self.shlibs_provides = None # self.shlibs_requires = None # TODO: Support checksum. # self.sum = None self.users = None self.version = None # TODO: Can Python packages be vital? # self.vital = None self.www = None def finalize_options(self): self.set_undefined_options('bdist', ('bdist_base', 'bdist_base')) self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) self.ensure_format('tgz') self.bdist_dir = os.path.join(self.bdist_base, 'pkg') self.install_dir = os.path.join(self.bdist_dir, 'root') self.finalize_manifest_options() def finalize_manifest_options(self): project = self.distribution self.ensure_string('abi', self.get_abi()) self.ensure_string('arch', self.get_arch()) self.ensure_categories(project) self.ensure_string('comment', project.get_description()) self.ensure_desc(project) self.ensure_string_list('groups') self.ensure_string('license', self.resolve_license(project)) self.ensure_string('maintainer', self.get_maintainer(project)) self.ensure_name(project) self.ensure_string('origin', self.get_default_origin(project)) self.ensure_prefix('/usr/local') self.ensure_string_list('provides') self.ensure_string_list('requires') self.ensure_scripts() self.ensure_string('version', project.get_version()) self.ensure_string_list('users') self.ensure_string('www', project.get_url()) self.ensure_options() self.ensure_deps() self.maybe_rename_console_scripts(project) def run(self): self.build_and_install() self.make_pkg(self.generate_manifest_content()) self.maybe_remove_temp(self.bdist_base) def build_and_install(self): if self.use_wheel: self.build_and_install_via_wheel() else: self.build_and_install_via_setuptools() def build_and_install_via_setuptools(self): # Basically, we need the intermediate results of bdist_dumb, # but since it's too monolithic and does the stuff that we would like # to avoid, here short copy-paste happens /: build = self.reinitialize_command('build', reinit_subcommands=1) build.build_base = self.bdist_base self.run_command('build') install = self.reinitialize_command('install', reinit_subcommands=1) install.prefix = self.prefix install.root = self.install_dir install.warn_dir = 0 self.run_command('install') def build_and_install_via_wheel(self): if not wheel_available: raise RuntimeError('The `wheel` package is not available.') build = self.reinitialize_command('build', reinit_subcommands=1) build.build_base = self.bdist_base bdist_wheel = self.reinitialize_command('bdist_wheel', reinit_subcommands=1) bdist_wheel.bdist_base = self.bdist_base bdist_wheel.keep_temp = True self.run_command('bdist_wheel') name = self.distribution.get_name() pip.wheel.move_wheel_files( name=self.name, req=WhlRequirement.parse('{}=={}'.format(name, self.version)), wheeldir=bdist_wheel.bdist_dir, root=self.install_dir, prefix=self.prefix, ) def generate_manifest_content(self): manifest = { 'abi': self.abi, 'arch': self.arch, 'categories': self.categories, 'comment': self.comment, 'deps': self.deps, 'desc': self.desc, 'directories': {}, 'files': {}, 'flatsize': 0, 'groups': self.groups, 'licenselogic': 'single', 'licenses': [self.license] if self.license else [], 'maintainer': self.maintainer, 'name': self.name, 'options': self.options, 'origin': self.origin, 'prefix': self.prefix, 'provides': self.provides, 'requires': self.requires, 'scripts': self.scripts, 'users': self.users, 'version': self.version, 'www': self.www, } mdirs = manifest['directories'] mfiles = manifest['files'] for real_file_path, install_path in self.iter_install_files(): with open(real_file_path, 'rb') as fh: data = fh.read() manifest['flatsize'] += len(data) mdirs[os.path.dirname(install_path)] = { 'gname': 'wheel', 'perm': '0755', 'uname': 'root', } mfiles[install_path] = { 'gname': 'wheel', 'perm': '0644', 'sum': hashlib.sha256(data).hexdigest(), 'uname': 'root', } # TODO: Should we keep UNKNOWN values? manifest = { key: value for key, value in manifest.items() if value and value != 'UNKNOWN' } if 'name' not in manifest: raise DistutilsOptionError('Project must have name defined') if 'version' not in manifest: raise DistutilsOptionError('Project must have version defined') if 'comment' not in manifest: raise DistutilsOptionError('Project must have description defined') if 'desc' not in manifest: raise DistutilsOptionError('Project must have long_description' ' defined') if 'maintainer' not in manifest: raise DistutilsOptionError('Project must have author or maintainer' ' defined') return manifest def make_pkg(self, manifest): manifest_path = self.make_manifest(manifest) compact_manifest_path = self.make_compact_manifest(manifest) files_paths = chain( [(manifest_path, os.path.basename(manifest_path)), (compact_manifest_path, os.path.basename(compact_manifest_path))], self.iter_install_files()) self.mkpath(self.dist_dir) tar_path = self.make_tar(files_paths) ext = self.format if ext != 'tar': compressor = self.get_compressor(ext) if compressor is None: raise RuntimeError('Format {} is not supported'.format(ext)) self.compress_tar(tar_path, ext, compressor) os.remove(tar_path) def make_manifest(self, content): path = os.path.join(self.bdist_dir, '+MANIFEST') with open(path, 'w') as fobj: json.dump(content, fobj, sort_keys=True, indent=4) return path def make_compact_manifest(self, content): path = os.path.join(self.bdist_dir, '+COMPACT_MANIFEST') compact_content = content.copy() compact_content.pop('directories') compact_content.pop('files') with open(path, 'w') as fobj: json.dump(compact_content, fobj, sort_keys=True, indent=4) return path def make_tar(self, files_paths): basename = '{}-{}.tar'.format(self.name, self.version) path = os.path.join(self.dist_dir, basename) seen = set() with tarfile.open(path, 'w') as tar: for file_path, tar_path in files_paths: tar_dir_path = os.path.dirname(tar_path) if tar_dir_path and tar_dir_path not in seen: tarinfo = tar.gettarinfo(os.path.dirname(file_path), tar_dir_path) tarinfo.name = tar_dir_path tar.addfile(tarinfo) seen.add(tar_dir_path) tarinfo = tar.gettarinfo(file_path, tar_path) tarinfo.name = tar_path with open(file_path, 'rb') as f: tar.addfile(tarinfo, f) return path def compress_tar(self, tar_path, ext, compressor): txx_path = tar_path.rsplit('.tar', 1)[0] + '.' + ext with compressor.open(txx_path, 'w') as txx: with open(tar_path, 'rb') as tar: txx.write(tar.read()) return txx_path def get_compressor(self, format): return self.compressor_for_format.get(format) def get_abi(self): if platform.system().lower() != 'freebsd': if not self.distribution.is_pure(): raise DistutilsOptionError( 'Unable to determine default ABI value' ' since bdist_pkg call happens not on FreeBSD system.' ' Please specify this value according the target system' ' for which you build this package.') return '*' return ':'.join(( platform.system(), # 10.1-STABLE-r273058 -> 10 platform.release().split('-', 1)[0].split('.')[0], # TODO: ensure that platform.machine() gives correct values platform.machine())) def get_arch(self): if platform.system().lower() != 'freebsd': if not self.distribution.is_pure(): raise DistutilsOptionError( 'Unable to determine default ARCH value' ' since bdist_pkg call happens not on FreeBSD system.' ' Please specify this value according the target system' ' for which you build this package.') return '*' return ':'.join(( platform.system(), # 10.1-STABLE-r273058 -> 10 platform.release().split('-', 1)[0].split('.')[0], # TODO: shouldn't there be a better way? 'x86:64' if platform.machine() == 'amd64' else 'x86:32')) def get_default_origin(self, project): return 'devel/py{}{}-{}'.format(sys.version_info[0], sys.version_info[1], project.get_name()) def get_maintainer(self, project): maintainer = '{} <{}>'.format(project.get_maintainer(), project.get_maintainer_email()) if maintainer == 'UNKNOWN <UNKNOWN>': # No explicit maintainer specified, use author contact instead maintainer = '{} <{}>'.format(project.get_author(), project.get_author_email()) return maintainer def resolve_license(self, project): # Thanks for this mapping goes to pytoport project py2freebsd_mapping = { 'agpl-3.0': 'AGPLv3', 'apache-2.0': 'APACHE20', 'artistic-2.0': 'ART20', 'bsd-2-clause': 'BSD2CLAUSE', 'bsd-3-clause-clear': 'BSD3CLAUSE', 'bsd-3-clause': 'BSD3CLAUSE', 'cc0-1.0': 'CC0-1.0', 'epl-1.0': 'EPL', 'gpl-2.0': 'GPLv2', 'gpl-3.0': 'GPLv3', 'isc': 'ISCL', 'lgpl-2.1': 'LGPL21', 'lgpl-3.0': 'LGPL3', 'mit': 'MIT', 'mpl-2.0': 'MPL', 'ofl-1.1': 'OFL11', } license = project.get_license() pkg_license = py2freebsd_mapping.get(license.lower()) if license != 'UNKNOWN' and pkg_license is None: self.warn('Unable to convert license %s to PKG naming' % license) return license return pkg_license def ensure_format(self, default): self.ensure_string('format', default) if self.format not in {'txz', 'tbz', 'tgz', 'tar'}: self.warn('Unknown format {!r}, falling back to {}' ''.format(self.format, default)) self.format = default def ensure_prefix(self, default=None): self.ensure_string('prefix', default) self.prefix = self.prefix.rstrip('/') def ensure_categories(self, project): self.categories = self.categories or project.get_keywords() self.ensure_string_list('categories') def ensure_deps(self): install_requires = set(self.distribution.install_requires or []) for option in self.selected_options: install_requires |= set(self.distribution.extras_require[option]) mapping = self.requirements_mapping or {} self.deps = self.deps or {} seen_deps = set([]) for python_dep, spec in mapping.items(): if not isinstance(python_dep, str): raise DistutilsOptionError('Invalid Python dependency: {}' ''.format(python_dep)) if python_dep not in install_requires: raise DistutilsOptionError('{} is not in install requires list' ''.format(python_dep)) if not isinstance(spec, dict): raise DistutilsOptionError('requirements_mapping items must be' ' dict, got {}'.format(repr(spec))) if set(spec) != {'origin', 'version', 'name'}: raise DistutilsOptionError('requirements_mapping items must' ' have "origin" and "version" keys,' ' got {}'.format(set(spec))) for key in {'origin', 'version', 'name'}: if not isinstance(spec[key], str): raise DistutilsOptionError('"{}" value must be string, got' ' {}'.format(key, spec[key])) self.deps[spec['name']] = { 'origin': spec['origin'], 'version': spec['version'] } seen_deps.add(python_dep) missing = seen_deps ^ install_requires if missing and self.use_pypi_deps: for item in missing: requirement = Requirement.parse(item) distribution = self.package_index.obtain(requirement) key = 'py{1}{2}-{0}'.format(distribution.key, *sys.version_info[:2]) self.deps[key] = { 'origin': 'pypi/py-{}'.format(distribution.key), 'version': distribution.version } elif missing: raise DistutilsOptionError('These packages are listed in install' ' requirements, but not in bdist_pkg' ' requirements mapping: {}' ''.format(', '.join(missing))) def ensure_desc(self, project): desc = project.get_long_description() desc = desc if desc != 'UKNOWN' else project.get_description() desc = self.cut_changelog(desc) self.ensure_string('desc', desc) def ensure_name(self, project): name = project.get_name() if self.with_py_prefix: name = 'py{}{}-{}'.format(sys.version_info[0], sys.version_info[1], name) self.ensure_string('name', name) def ensure_options(self): provided_options = set(self.distribution.extras_require or {}) self.selected_options = set(self.selected_options or []) unknown_options = self.selected_options - provided_options if not unknown_options: self.options = { option: option in self.selected_options for option in provided_options } else: raise DistutilsOptionError('Unknown extras selected: {}' ''.format(', '.join(unknown_options))) def ensure_scripts(self): if self.scripts is None: return if not isinstance(self.scripts, dict): raise DistutilsOptionError('scripts must be a dict, got {}' ''.format(self.scripts)) valid_keys = { 'pre-install', 'post-install', 'install', 'pre-deinstall', 'post-deinstall', 'deinstall', 'pre-upgrade', 'post-upgrade', 'upgrade', } bad_keys = [key for key in self.scripts if key not in valid_keys] if bad_keys: raise DistutilsOptionError('invalid scripts: {}' ''.format(', '.join(bad_keys))) bad_keys = [ key for key, value in self.scripts.items() if not isinstance(value, str) ] if bad_keys: raise DistutilsOptionError('invalid scripts: {}' ''.format(', '.join(bad_keys))) def iter_install_files(self): for root, dirs, files in os.walk(self.install_dir): for file in files: reldir = os.path.relpath(root, self.install_dir) install_path = '/' + os.path.join(reldir, file) install_path = install_path.replace(self.prefix + '/lib64/', self.prefix + '/lib/') yield os.path.join(root, file), install_path def maybe_remove_temp(self, path): if self.keep_temp: return if path is None: return if os.path.exists(path): shutil.rmtree(path) def maybe_rename_console_scripts(self, project): if not self.with_py_prefix: return if not project.entry_points: return console_scripts = project.entry_points.get('console_scripts') if console_scripts is None: return prefixed_console_scripts = [] for script in console_scripts: name, callback = script.split('=') name = '{}{}.{}'.format(name.strip(), *sys.version_info[:2]) prefixed_console_scripts.append('{} = {}'.format( name, callback.strip())) project.entry_points['console_scripts'] = prefixed_console_scripts def cut_changelog(self, desc): def match_changelog_header(line): words = re.findall(r'\b\w+\b', line.lower()) if len(words) != 1: return True if 'changelog' in words or 'changes' in words: return False return True return '\n'.join(takewhile(match_changelog_header, desc.splitlines()))
def runit(): # process command-line options bool_opts = map(translate_longopt, stdeb_cmd_bool_opts) bool_opts.append("process-dependencies") parser = FancyGetopt(stdeb_cmdline_opts + [("help", "h", "show detailed help message")] + EXTRA_OPTS) optobj = OptObj() args = parser.getopt(object=optobj) idx = PackageIndex() for option in optobj.__dict__: value = getattr(optobj, option) is_string = type(value) == str if option in bool_opts and is_string: setattr(optobj, option, strtobool(value)) if hasattr(optobj, "help"): print USAGE parser.set_option_table(stdeb_cmdline_opts + EXTRA_OPTS) parser.print_help("Options:") return 0 if len(args) != 1: log.error("not given single argument (distfile), args=%r", args) print USAGE return 1 sdist_file = args[0] package = None final_dist_dir = optobj.__dict__.get("dist_dir", "deb_dist") tmp_dist_dir = os.path.join(final_dist_dir, "tmp_py2dsc") if os.path.exists(tmp_dist_dir): shutil.rmtree(tmp_dist_dir) os.makedirs(tmp_dist_dir) if not os.path.isfile(sdist_file): for ext in EXTENSIONS: if sdist_file.endswith(ext): raise IOError, "File not found" package = Requirement.parse(sdist_file) log.info("Package %s not found, trying PyPI..." % sdist_file) dist = idx.fetch_distribution(package, final_dist_dir, force_scan=True, source=True) if hasattr(dist, "location"): sdist_file = dist.location else: raise Exception, "Distribution not found on PyPi" log.info("Got %s", sdist_file) dist = list(distros_for_filename(sdist_file))[0] idx.scan_egg_links(dist.location) package = idx.obtain(Requirement.parse(dist.project_name)) if hasattr(optobj, "process_dependencies"): if bool(int(getattr(optobj, "process_dependencies"))): backup_argv = sys.argv[:] oldargv = sys.argv[:] oldargv.pop(-1) if package.requires(): log.info("Processing package dependencies for %s", package) for req in package.requires(): # print >> sys.stderr new_argv = oldargv + ["%s" % req] log.info("Bulding dependency package %s", req) log.info(" running '%s'", " ".join(new_argv)) sys.argv = new_argv runit() # print >> sys.stderr if package.requires(): log.info("Completed building dependencies " "for %s, continuing...", package) sys.argv = backup_argv if package is not None and hasattr(optobj, "extra_cfg_file"): # Allow one to have patch-files setup on config file for example local_parser = SafeConfigParser() local_parser.readfp(open(optobj.__dict__.get("extra_cfg_file"))) if local_parser.has_section(package.project_name): for opt in local_parser.options(package.project_name): _opt = opt.replace("_", "-") if parser.has_option(_opt) or parser.has_option(_opt + "="): setattr(optobj, opt, local_parser.get(package.project_name, opt)) patch_file = optobj.__dict__.get("patch_file", None) patch_level = int(optobj.__dict__.get("patch_level", 0)) patch_posix = int(optobj.__dict__.get("patch_posix", 0)) expand_dir = os.path.join(tmp_dist_dir, "stdeb_tmp") if os.path.exists(expand_dir): shutil.rmtree(expand_dir) if not os.path.exists(tmp_dist_dir): os.mkdir(tmp_dist_dir) os.mkdir(expand_dir) expand_sdist_file(os.path.abspath(sdist_file), cwd=expand_dir) # now the sdist package is expanded in expand_dir expanded_root_files = os.listdir(expand_dir) assert len(expanded_root_files) == 1 repackaged_dirname = expanded_root_files[0] fullpath_repackaged_dirname = os.path.join(tmp_dist_dir, repackaged_dirname) base_dir = os.path.join(expand_dir, expanded_root_files[0]) if os.path.exists(fullpath_repackaged_dirname): # prevent weird build errors if this dir exists shutil.rmtree(fullpath_repackaged_dirname) os.renames(base_dir, fullpath_repackaged_dirname) del base_dir # no longer useful ############################################## if patch_file is not None: log.info("py2dsc applying patch %s", patch_file) apply_patch(patch_file, posix=patch_posix, level=patch_level, cwd=fullpath_repackaged_dirname) patch_already_applied = 1 else: patch_already_applied = 0 ############################################## abs_dist_dir = os.path.abspath(final_dist_dir) extra_args = [] for long in parser.long_opts: if long in ["dist-dir=", "patch-file=", "process-dependencies"]: continue # dealt with by this invocation attr = parser.get_attr_name(long).rstrip("=") if hasattr(optobj, attr): val = getattr(optobj, attr) if attr == "extra_cfg_file": val = os.path.abspath(val) if long in bool_opts or long.replace("-", "_") in bool_opts: extra_args.append("--%s" % long) else: extra_args.append("--" + long + str(val)) if patch_already_applied == 1: extra_args.append("--patch-already-applied") args = [ sys.executable, "-c", "import stdeb, sys; f='setup.py'; " + "sys.argv[0]=f; execfile(f,{'__file__':f,'__name__':'__main__'})", "sdist_dsc", "--dist-dir=%s" % abs_dist_dir, "--use-premade-distfile=%s" % os.path.abspath(sdist_file), ] + extra_args log.info("-=" * 35 + "-") # print >> sys.stderr, '-='*20 # print >> sys.stderr, "Note that the .cfg file(s), if present, have not "\ # "been read at this stage. If options are necessary, pass them from "\ # "the command line" log.info("running the following command in directory: %s\n%s", fullpath_repackaged_dirname, " ".join(args)) log.info("-=" * 35 + "-") try: returncode = subprocess.call(args, cwd=fullpath_repackaged_dirname) except: log.error("ERROR running: %s", " ".join(args)) log.error("ERROR in %s", fullpath_repackaged_dirname) raise if returncode: log.error("ERROR running: %s", " ".join(args)) log.error("ERROR in %s", fullpath_repackaged_dirname) # log.error(' stderr: %s'res.stderr.read()) # print >> sys.stderr, 'ERROR running: %s'%(' '.join(args),) # print >> sys.stderr, res.stderr.read() return returncode # raise RuntimeError('returncode %d'%returncode) # result = res.stdout.read().strip() shutil.rmtree(tmp_dist_dir) return returncode