def get_license_name(package, pkginfo, no_prompt=False, data=None): """Responsible to return the license name :param str package: Package's name :param dict pkginfo:Package information :param no_prompt: If Prompt is not enabled :param dict data: Data :return str: License name """ license_classifier = "License :: OSI Approved :: " data_classifier = data.get("classifiers", []) if data else [] pkg_classifier = pkginfo.get('classifiers', data_classifier) pkg_classifier = pkg_classifier if pkg_classifier else data_classifier licenses = [ classifier.split(license_classifier, 1)[1] for classifier in pkg_classifier if classifier.startswith(license_classifier) ] if licenses: return ' or '.join(licenses) if pkginfo.get('license'): license_name = pkginfo['license'] elif data and 'license' in data: license_name = data['license'] else: license_name = None if license_name: if no_prompt: return license_name elif '\n' not in license_name: print('Using "%s" for the license' % license_name) else: # Some projects put the whole license text in this field print("This is the license for %s" % package) print() print(license_name) print() license_name = input("What license string should I use? ") elif no_prompt: license_name = "UNKNOWN" else: license_name = input( "No license could be found for %s on PyPI or in the source. " "What license should I use? " % package ) return license_name
def get_download_data(client, package, version, is_url, all_urls, noprompt, manual_url): data = client.release_data(package, version) if not is_url else None urls = client.release_urls(package, version) if not is_url else [package] if not is_url and not all_urls: # Try to find source urls urls = [url for url in urls if url['python_version'] == 'source'] if not urls: if 'download_url' in data: urls = [defaultdict(str, {'url': data['download_url']})] if not urls[0]['url']: # The package doesn't have a url, or maybe it only has a wheel. sys.exit("Error: Could not build recipe for %s. " "Could not find any valid urls." % package) U = parse_url(urls[0]['url']) if not U.path: sys.exit("Error: Could not parse url for %s: %s" % (package, U)) urls[0]['filename'] = U.path.rsplit('/')[-1] fragment = U.fragment or '' if fragment.startswith('md5='): md5 = fragment[len('md5='):] else: md5 = '' else: sys.exit("Error: No source urls found for %s" % package) if len(urls) > 1 and not noprompt: print("More than one source version is available for %s:" % package) if manual_url: for i, url in enumerate(urls): print("%d: %s (%s) %s" % (i, url['url'], human_bytes(url['size']), url['comment_text'])) n = int(input("which version should i use? ")) else: print("Using the one with the least source size") print("use --manual-url to override this behavior") _, n = min([(url['size'], i) for (i, url) in enumerate(urls)]) else: n = 0 if not is_url: print("Using url %s (%s) for %s." % (urls[n]['url'], human_bytes(urls[n]['size'] or 0), package)) pypiurl = urls[n]['url'] md5 = urls[n]['md5_digest'] filename = urls[n]['filename'] or 'package' else: print("Using url %s" % package) pypiurl = package U = parse_url(package) if U.fragment and U.fragment.startswith('md5='): md5 = U.fragment[len('md5='):] else: md5 = '' # TODO: 'package' won't work with unpack() filename = U.path.rsplit('/', 1)[-1] or 'package' return (data, pypiurl, filename, md5)
def get_package_metadata(package, d, data, output_dir, python_version, all_extras, recursive, created_recipes, noarch_python, noprompt, packages, extra_specs, config, setup_options): print("Downloading %s" % package) print("PyPI URL: ", d['pypiurl']) pkginfo = get_pkginfo(package, filename=d['filename'], pypiurl=d['pypiurl'], digest=d['digest'], python_version=python_version, extra_specs=extra_specs, setup_options=setup_options, config=config) setuptools_run = False # Look at the entry_points and construct console_script and # gui_scripts entry_points for conda entry_points = pkginfo.get('entry_points', []) if entry_points: if isinstance(entry_points, str): # makes sure it is left-shifted newstr = "\n".join(x.strip() for x in entry_points.splitlines()) _config = configparser.ConfigParser() entry_points = {} try: _config.readfp(StringIO(newstr)) except Exception as err: print("WARNING: entry-points not understood: ", err) print("The string was", newstr) entry_points = pkginfo['entry_points'] else: setuptools_run = True for section in _config.sections(): if section in ['console_scripts', 'gui_scripts']: value = [ '%s=%s' % (option, _config.get(section, option)) for option in _config.options(section) ] entry_points[section] = value if not isinstance(entry_points, dict): print("WARNING: Could not add entry points. They were:") print(entry_points) else: cs = entry_points.get('console_scripts', []) gs = entry_points.get('gui_scripts', []) if isinstance(cs, string_types): cs = [cs] elif cs and isinstance(cs, list) and isinstance(cs[0], list): # We can have lists of lists here cs = [item for sublist in [s for s in cs] for item in sublist] if isinstance(gs, string_types): gs = [gs] elif gs and isinstance(gs, list) and isinstance(gs[0], list): gs = [item for sublist in [s for s in gs] for item in sublist] # We have *other* kinds of entry-points so we need # setuptools at run-time if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}: setuptools_run = True # TODO: Use pythonw for gui scripts entry_list = (cs + gs) if len(cs + gs) != 0: d['entry_points'] = entry_list d['test_commands'] = make_entry_tests(entry_list) requires = get_requirements(package, pkginfo, all_extras=all_extras) if requires or setuptools_run: deps = [] if setuptools_run: deps.append('setuptools') for deptext in requires: if isinstance(deptext, string_types): deptext = deptext.splitlines() # Every item may be a single requirement # or a multiline requirements string... for dep in deptext: # ... and may also contain comments... dep = dep.split('#')[0].strip() if dep: # ... and empty (or comment only) lines dep, marker = parse_dep_with_env_marker(dep) spec = spec_from_line(dep) if spec is None: sys.exit("Error: Could not parse: %s" % dep) if '~' in spec: version = spec.split()[-1] tilde_version = '~ {}'.format(version) pin_compatible = convert_version(version) spec = spec.replace(tilde_version, pin_compatible) if marker: spec = ' '.join((spec, marker)) deps.append(spec) if 'setuptools' in deps: setuptools_run = False d['build_depends'] = ['pip'] + deps # Never add setuptools to runtime dependencies. d['run_depends'] = deps if recursive: for dep in deps: dep = dep.split()[0] if not exists(join(output_dir, dep)): if dep not in created_recipes: packages.append(dep) if 'packagename' not in d: d['packagename'] = pkginfo['name'].lower() if d['version'] == 'UNKNOWN': d['version'] = pkginfo['version'] if pkginfo.get('packages'): deps = set(pkginfo['packages']) if d['import_tests']: if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER': olddeps = [] else: olddeps = [x for x in d['import_tests'].split() if x != '-'] deps = set(olddeps) | deps d['import_tests'] = sorted(deps) d['tests_require'] = sorted([ spec_from_line(pkg) for pkg in ensure_list(pkginfo['tests_require']) ]) if pkginfo.get('home'): d['home'] = pkginfo['home'] else: if data and 'home' in data: d['home'] = data['home'] else: d['home'] = "The package home page" if pkginfo.get('summary'): if 'summary' in d and not d['summary']: # Need something here, use what the package had d['summary'] = pkginfo['summary'] else: d['summary'] = "Summary of the package" license_classifier = "License :: OSI Approved :: " if pkginfo.get('classifiers'): licenses = [ classifier.split(license_classifier, 1)[1] for classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier) ] elif data and 'classifiers' in data: licenses = [ classifier.split(license_classifier, 1)[1] for classifier in data['classifiers'] if classifier.startswith(license_classifier) ] else: licenses = [] if not licenses: if pkginfo.get('license'): license_name = pkginfo['license'] elif data and 'license' in data: license_name = data['license'] else: license_name = None if license_name: if noprompt: pass elif '\n' not in license_name: print('Using "%s" for the license' % license_name) else: # Some projects put the whole license text in this field print("This is the license for %s" % package) print() print(license_name) print() license_name = input("What license string should I use? ") else: if noprompt: license_name = "UNKNOWN" else: license_name = input( ("No license could be found for %s on " + "PyPI or in the source. What license should I use? ") % package) else: license_name = ' or '.join(licenses) # remove the word license from the license clean_license_name = re.subn('(.*)\s+license', r'\1', license_name, flags=re.IGNORECASE)[0] d['license'] = clean_license_name d['license_family'] = guess_license_family(license_name, allowed_license_families) if 'new_hash_value' in pkginfo: d['digest'] = pkginfo['new_hash_value']
def get_download_data(pypi_data, package, version, is_url, all_urls, noprompt, manual_url): """ Get at least one valid *source* download URL or fail. Returns ------- data : dict Summary of package information pypiurl : str Download URL of package, which may or may not actually be from PyPI. filename : str Name of file; used to check cache digest : dict Key is type of checksum, value is the checksum. """ data = pypi_data['info'] if not is_url else {} # PyPI will typically have several downloads (source, wheels) for one # package/version. urls = [url for url in pypi_data['releases'][version] ] if not is_url else [package] if not is_url and not all_urls: # Try to find source urls urls = [url for url in urls if url['packagetype'] == 'sdist'] if not urls: # Try harder for a download location if data.get('download_url'): urls = [defaultdict(str, {'url': data['download_url']})] if not urls[0]['url']: # The package doesn't have a url, or maybe it only has a wheel. sys.exit("Error: Could not build recipe for %s. " "Could not find any valid urls." % package) U = parse_url(urls[0]['url']) if not U.path: sys.exit("Error: Could not parse url for %s: %s" % (package, U)) urls[0]['filename'] = U.path.rsplit('/')[-1] fragment = U.fragment or '' digest = fragment.split("=") else: sys.exit("Error: No source urls found for %s" % package) if len(urls) > 1 and not noprompt: print("More than one source version is available for %s:" % package) if manual_url: for i, url in enumerate(urls): print("%d: %s (%s) %s" % (i, url['url'], human_bytes( url['size']), url['comment_text'])) n = int(input("which version should i use? ")) else: print("Using the one with the least source size") print("use --manual-url to override this behavior") _, n = min([(url['size'], i) for (i, url) in enumerate(urls)]) else: n = 0 if not is_url: # Found a location from PyPI. url = urls[n] pypiurl = url['url'] print("Using url %s (%s) for %s." % (pypiurl, human_bytes(url['size'] or 0), package)) if url['digests']['sha256']: digest = ('sha256', url['digests']['sha256']) else: # That didn't work, even though as of 7/17/2017 some packages # have a 'digests' entry. # As a last-ditch effort, try for the md5_digest entry. digest = () filename = url['filename'] or 'package' else: # User provided a URL, try to use it. print("Using url %s" % package) pypiurl = package U = parse_url(package) digest = U.fragment.split("=") # TODO: 'package' won't work with unpack() filename = U.path.rsplit('/', 1)[-1] or 'package' return (data, pypiurl, filename, digest)
def get_package_metadata(package, d, data, output_dir, python_version, all_extras, recursive, created_recipes, noarch_python, noprompt, packages, config, setup_options): print("Downloading %s" % package) pkginfo = get_pkginfo(package, filename=d['filename'], pypiurl=d['pypiurl'], md5=d['md5'], python_version=python_version, setup_options=setup_options, config=config) setuptools_build = pkginfo['setuptools'] setuptools_run = False # Look at the entry_points and construct console_script and # gui_scripts entry_points for conda entry_points = pkginfo['entry_points'] if entry_points: if isinstance(entry_points, str): # makes sure it is left-shifted newstr = "\n".join(x.strip() for x in entry_points.splitlines()) _config = configparser.ConfigParser() entry_points = {} try: _config.readfp(StringIO(newstr)) except Exception as err: print("WARNING: entry-points not understood: ", err) print("The string was", newstr) entry_points = pkginfo['entry_points'] else: setuptools_run = True for section in config.sections(): if section in ['console_scripts', 'gui_scripts']: value = ['%s=%s' % (option, _config.get(section, option)) for option in config.options(section)] entry_points[section] = value if not isinstance(entry_points, dict): print("WARNING: Could not add entry points. They were:") print(entry_points) else: cs = entry_points.get('console_scripts', []) gs = entry_points.get('gui_scripts', []) if isinstance(cs, string_types): cs = [cs] if isinstance(gs, string_types): gs = [gs] # We have *other* kinds of entry-points so we need # setuptools at run-time if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}: setuptools_build = True setuptools_run = True # TODO: Use pythonw for gui scripts entry_list = (cs + gs) if len(cs + gs) != 0: d['entry_points'] = INDENT.join([''] + entry_list) d['entry_comment'] = '' d['build_comment'] = '' d['test_commands'] = INDENT.join([''] + make_entry_tests(entry_list)) requires = get_requirements(package, pkginfo, all_extras=all_extras) if requires or setuptools_build or setuptools_run: deps = [] if setuptools_run: deps.append('setuptools') for deptext in requires: if isinstance(deptext, string_types): deptext = deptext.splitlines() # Every item may be a single requirement # or a multiline requirements string... for dep in deptext: # ... and may also contain comments... dep = dep.split('#')[0].strip() if dep: # ... and empty (or comment only) lines spec = spec_from_line(dep) if spec is None: sys.exit("Error: Could not parse: %s" % dep) deps.append(spec) if 'setuptools' in deps: setuptools_build = False setuptools_run = False d['egg_comment'] = '' d['build_comment'] = '' d['build_depends'] = INDENT.join([''] + ['setuptools'] * setuptools_build + deps) d['run_depends'] = INDENT.join([''] + ['setuptools'] * setuptools_run + deps) if recursive: for dep in deps: dep = dep.split()[0] if not exists(join(output_dir, dep)): if dep not in created_recipes: packages.append(dep) if noarch_python: d['build_comment'] = '' d['noarch_python_comment'] = '' if 'packagename' not in d: d['packagename'] = pkginfo['name'].lower() if d['version'] == 'UNKNOWN': d['version'] = pkginfo['version'] if pkginfo['packages']: deps = set(pkginfo['packages']) if d['import_tests']: if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER': olddeps = [] else: olddeps = [x for x in d['import_tests'].split() if x != '-'] deps = set(olddeps) | deps d['import_tests'] = INDENT.join(sorted(deps)) d['import_comment'] = '' d['tests_require'] = INDENT.join(sorted([spec_from_line(pkg) for pkg in pkginfo['tests_require']])) if pkginfo['homeurl'] is not None: d['homeurl'] = pkginfo['homeurl'] else: if data and 'homeurl' in data: d['homeurl'] = data['homeurl'] else: d['homeurl'] = "The package home page" d['home_comment'] = '#' if pkginfo['summary']: d['summary'] = repr(pkginfo['summary']) else: if data: d['summary'] = repr(data['summary']) else: d['summary'] = "Summary of the package" d['summary_comment'] = '#' if d['summary'].startswith("u'") or d['summary'].startswith('u"'): d['summary'] = d['summary'][1:] license_classifier = "License :: OSI Approved :: " if pkginfo['classifiers']: licenses = [classifier.split(license_classifier, 1)[1] for classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier)] elif data and 'classifiers' in data: licenses = [classifier.split(license_classifier, 1)[1] for classifier in data['classifiers'] if classifier.startswith(license_classifier)] else: licenses = [] if not licenses: if pkginfo['license']: license_name = pkginfo['license'] elif data and 'license' in data: license_name = data['license'] else: license_name = None if license_name: if noprompt: pass elif '\n' not in license_name: print('Using "%s" for the license' % license_name) else: # Some projects put the whole license text in this field print("This is the license for %s" % package) print() print(license_name) print() license_name = input("What license string should I use? ") else: if noprompt: license_name = "UNKNOWN" else: license_name = input(("No license could be found for %s on " + "PyPI or in the source. What license should I use? ") % package) else: license_name = ' or '.join(licenses) d['license'] = license_name d['license_family'] = guess_license_family(license_name, allowed_license_families)
def get_download_data(pypi_data, package, version, is_url, all_urls, noprompt, manual_url): """ Get at least one valid *source* download URL or fail. Returns ------- data : dict Summary of package information pypiurl : str Download URL of package, which may or may not actually be from PyPI. filename : str Name of file; used to check cache digest : dict Key is type of checksum, value is the checksum. """ data = pypi_data['info'] if not is_url else {} # PyPI will typically have several downloads (source, wheels) for one # package/version. urls = [url for url in pypi_data['releases'][version]] if not is_url else [package] if not is_url and not all_urls: # Try to find source urls urls = [url for url in urls if url['packagetype'] == 'sdist'] if not urls: # Try harder for a download location if data.get('download_url'): urls = [defaultdict(str, {'url': data['download_url']})] if not urls[0]['url']: # The package doesn't have a url, or maybe it only has a wheel. sys.exit("Error: Could not build recipe for %s. " "Could not find any valid urls." % package) U = parse_url(urls[0]['url']) if not U.path: sys.exit("Error: Could not parse url for %s: %s" % (package, U)) urls[0]['filename'] = U.path.rsplit('/')[-1] fragment = U.fragment or '' digest = digest_from_fragment(fragment) else: sys.exit("Error: No source urls found for %s" % package) if len(urls) > 1 and not noprompt: print("More than one source version is available for %s:" % package) if manual_url: for i, url in enumerate(urls): print("%d: %s (%s) %s" % (i, url['url'], human_bytes(url['size']), url['comment_text'])) n = int(input("which version should i use? ")) else: print("Using the one with the least source size") print("use --manual-url to override this behavior") _, n = min([(url['size'], i) for (i, url) in enumerate(urls)]) else: n = 0 if not is_url: # Found a location from PyPI. url = urls[n] pypiurl = url['url'] print("Using url %s (%s) for %s." % (pypiurl, human_bytes(url['size'] or 0), package)) # List of digests we might get in order of preference for p in POSSIBLE_DIGESTS: try: if url['digests'][p]: digest = (p, url['digests'][p]) break except KeyError: continue else: # That didn't work, even though as of 7/17/2017 some packages # have a 'digests' entry. # As a last-ditch effort, try for the md5_digest entry. try: digest = ('md5', url['md5_digest']) except KeyError: # Give up digest = () filename = url['filename'] or 'package' else: # User provided a URL, try to use it. print("Using url %s" % package) pypiurl = package U = parse_url(package) digest = digest_from_fragment(U.fragment) # TODO: 'package' won't work with unpack() filename = U.path.rsplit('/', 1)[-1] or 'package' return (data, pypiurl, filename, digest)