def _build_std_library():
python_version = sys.version_info
libraries = stdlib_list(
'{0}.{1}'.format(
python_version[0],
python_version[1],
)
)
fake_std_libraries = [DottedName(x) for x in libraries]
return fake_std_libraries
def import_distributions(specs, distribution_blacklist=DISTRIBUTION_BLACKLIST):
distributions_modules = collections.OrderedDict()
distributions = collect_distributions(specs, distribution_blacklist)
for requirement, distribution in distributions.items():
if requirement.startswith('Python=='):
python_version = requirement.partition('==')[2]
# stdlib_list supports short versions and only a selected list of long versions
python_short_version = python_version[:3]
module_names = stdlib_list.stdlib_list(python_short_version)
elif distribution.has_metadata('top_level.txt'):
module_names = distribution.get_metadata('top_level.txt').splitlines()
else:
logger.info("Package %r has no top_level.txt. Guessing module name is %r.",
requirement, distribution.project_name)
module_names = [distribution.project_name]
modules = import_modules(module_names, requirement=requirement)
distributions_modules[requirement] = list(modules.keys())
return distributions_modules
def pystdlib():
"""Return a set of all module-names in the Python standard library.
"""
curver = '.'.join(str(x) for x in sys.version_info[:2])
return (set(stdlib_list.stdlib_list(curver)) | {
'_LWPCookieJar', '_MozillaCookieJar', '_abcoll', 'email._parseaddr',
'email.base64mime', 'email.feedparser', 'email.quoprimime',
'encodings', 'genericpath', 'ntpath', 'nturl2path', 'os2emxpath',
'posixpath', 'sre_compile', 'sre_parse', 'unittest.case',
'unittest.loader', 'unittest.main', 'unittest.result',
'unittest.runner', 'unittest.signals', 'unittest.suite',
'unittest.util', '_threading_local', 'sre_constants', 'strop',
'repr', 'opcode', 'nt', 'encodings.aliases',
'_bisect', '_codecs', '_collections', '_functools', '_hashlib',
'_heapq', '_io', '_locale', '_LWPCookieJar', '_md5',
'_MozillaCookieJar', '_random', '_sha', '_sha256', '_sha512',
'_socket', '_sre', '_ssl', '_struct', '_subprocess',
'_threading_local', '_warnings', '_weakref', '_weakrefset',
'_winreg'
}) - {'__main__'}
# standard library
import standardlibrary
# list standard libary modules
import sys
print(sys.builtin_module_names[30:35])
# list standard libary modules
from stdlib_list import stdlib_list
libraries = stdlib_list("3.5")
print(libraries[4:10])
# find operating system type
print(sys.platform)
import argparse
# boolean
parser.add_argument('-a', action="store_true", default=False)
from warehouse.packaging.interfaces import IFileStorage
from warehouse.packaging.models import (
Project, Release, Dependency, DependencyKind, Role, File, Filename,
JournalEntry, BlacklistedProject,
)
from warehouse.utils import http
MAX_FILESIZE = 60 * 1024 * 1024 # 60M
MAX_SIGSIZE = 8 * 1024 # 8K
PATH_HASHER = "blake2_256"
STDLIB_PROHIBITTED = {
packaging.utils.canonicalize_name(s.rstrip('-_.').lstrip('-_.'))
for s in chain.from_iterable(stdlib_list.stdlib_list(version)
for version in stdlib_list.short_versions)
}
# Wheel platform checking
# These platforms can be handled by a simple static list:
_allowed_platforms = {
"any",
"win32", "win_amd64", "win_ia64",
"manylinux1_x86_64", "manylinux1_i686",
"linux_armv6l", "linux_armv7l",
}
# macosx is a little more complicated:
_macosx_platform_re = re.compile("macosx_10_(\d+)+_(?P<arch>.*)")
_macosx_arches = {
"ppc", "ppc64",
def get_list_stdli(self):
return stdlib_list("2.7")
}
PACKAGE_REGEX = re.compile(
r"^(?:--.+\s)?([-_\.\w\d\[\]]+)(==|>=|<=|~=|!=|<|>|===)*(.*)$")
PIP_REGEX = re.compile(
r"^(--.+\s)?([-_\.\w\d]+.*(?:==|>=|<=|~=|!=|<|>|===)?.*$)")
PIP_VERSION_RANGE_SEPARATOR = re.compile(r"^(==|>=|<=|~=|!=|<|>|===)?(.*)$")
SUPPORTED_PYTHON_TUPLES = [
REQUIRED_PYTHON_VER[:2],
tuple(map(operator.add, REQUIRED_PYTHON_VER, (0, 1, 0)))[:2],
]
SUPPORTED_PYTHON_VERSIONS = [
".".join(map(str, version_tuple))
for version_tuple in SUPPORTED_PYTHON_TUPLES
]
STD_LIBS = {
version: set(stdlib_list(version))
for version in SUPPORTED_PYTHON_VERSIONS
}
PIPDEPTREE_CACHE = None
IGNORE_VIOLATIONS = {
# Still has standard library requirements.
"acmeda",
"blink",
"ezviz",
"hdmi_cec",
"juicenet",
"lupusec",
"rainbird",
"slide",
"suez_water",
def check_more_builtins(name):
libraries = stdlib_list("3.7")
if name in libraries:
return True
return False
MAX_SIGSIZE = 8 * 1024 # 8K
PATH_HASHER = "blake2_256"
def namespace_stdlib_list(module_list):
for module_name in module_list:
parts = module_name.split(".")
for i, part in enumerate(parts):
yield ".".join(parts[: i + 1])
STDLIB_PROHIBITTED = {
packaging.utils.canonicalize_name(s.rstrip("-_.").lstrip("-_."))
for s in chain.from_iterable(
namespace_stdlib_list(stdlib_list.stdlib_list(version))
for version in stdlib_list.short_versions
)
}
# Wheel platform checking
# Note: defining new platform ABI compatibility tags that don't
# have a python.org binary release to anchor them is a
# complex task that needs more than just OS+architecture info.
# For Linux specifically, the platform ABI is defined by each
# individual distro version, so wheels built on one version may
# not even work on older versions of the same distro, let alone
# a completely different distro.
#
# That means new entries should only be added given an
def kw_only(**options):
print(options)
foo(x=1)
# standard library
import standardlibrary
# list standard libary modules
import sys
print(sys.builtin_module_names[30:35])
# list standard libary modules
from stdlib_list import stdlib_list
libraries = stdlib_list("3.5")
print(libraries[4:10])
# find operating system type
print(sys.platform)
import argparse
# boolean
parser.add_argument('-a', action="store_true", default=False)
import bisect
a = [12, 35, 4, 58, 6, 48, 98]
a = sorted(a)
index = bisect.bisect(a, 50)
def collect_stdlib_distributions():
# use Python long version number in distribution_spec
distribution_spec = 'Python==%d.%d.%d' % sys.version_info[:3]
# use Python short version number for stdlib_list as it supports only a few long versions
distribution_module_names = stdlib_list.stdlib_list('%d.%d' % sys.version_info[:2])
yield distribution_spec, distribution_module_names
import argparse
import os.path as o
from pathlib import Path
import itertools
from stdlib_list import stdlib_list
libraries = stdlib_list("3.8")
def readfile(file):
f = open(path(file), "r").readlines()
return f
def extract_modules(file, results={}):
packages = [parse(x) for x in readfile(file) if 'import' in x]
if packages:
results[file] = packages
for p in packages:
if p not in libraries:
extract_modules(p)
return results
def parse(statement):
return statement.rstrip().split()[1].replace('.', '/')
def format(mod, main):
m = extract_modules(main)
unique_mods = set(
# vim: expandtab tabstop=4 shiftwidth=4
from stdlib_list import stdlib_list
def good_lib(lib):
if '.' in lib or '__' in lib or lib.startswith('_'):
return False
return True
if __name__ == "__main__":
libs2 = sorted(list(set([ lib.lower() for lib in stdlib_list("2.7") if good_lib(lib) ])))
libs3 = sorted(list(set([ lib.lower() for lib in stdlib_list("3.6") if good_lib(lib) ])))
with open('ipydeps/data/libs2.txt', 'w') as f:
for lib in libs2:
f.write(lib + '\n')
with open('ipydeps/data/libs3.txt', 'w') as f:
for lib in libs3:
f.write(lib + '\n')
# flake8: NOQA
# encoding: UTF-8
print("\n\n\n\n")
import os
import sys
# import_standard_library
try:
from stdlib_list import stdlib_list
version = sys.version[:3]
libs = stdlib_list(version)
for lib in libs:
statement = 'import {}'.format(lib)
try:
exec(statement)
except ImportError:
pass
except:
pass
# stdlib extra imports
from collections import *
from datetime import *
from datetime import datetime as dt
now = dt.now()
from conda_forge_tick.xonsh_utils import indir, env
RUNTIME_MINUTES = 45
IGNORE_STUBS = ["doc", "example", "demo", "test", "unit_tests", "testing"]
IGNORE_TEMPLATES = ["*/{z}/*", "*/{z}s/*"]
DEPFINDER_IGNORE = []
for k in IGNORE_STUBS:
for tmpl in IGNORE_TEMPLATES:
DEPFINDER_IGNORE.append(tmpl.format(z=k))
DEPFINDER_IGNORE += ["*testdir/*", "*conftest*", "*/test.py", "*/versioneer.py"]
BUILTINS = set().union(
# Some libs support older python versions, we don't want their std lib
# entries in our diff though
*[set(stdlib_list(k)) for k in ["2.7", "3.5", "3.6", "3.7"]]
)
STATIC_EXCLUDES = (
{
"python",
"setuptools",
"pip",
"versioneer",
# not a real dep
"cross-python",
}
| BUILTINS
| set(COMPILER_STUBS_WITH_STRONG_EXPORTS)
)
from pypidb._compat import PY2
from pypidb._db import Database, multipackage_repos
from pypidb._github import check_repo as check_github_repo, get_repo_setuppy
from pypidb._github import GitHubAPIMessage
from pypidb._pypi import IncompletePackageMetadata, InvalidPackage
from pypidb._rules import DefaultRule, Rule, rules
from pypidb._similarity import _compute_similarity, normalize
from tests.data import (
mismatch,
missing_repos,
setuppy_mismatches,
setuppy_mismatches_xstatic,
wrong_result,
)
_stdlib_all = stdlib_list()
_stdlib = set(i.split(".")[0] for i in _stdlib_all)
USE_SEPARATE_INSTANCE = (
True) # This can only be disabled if only one set of tests are run
web_session = get_file_cache("web")
def normalise_list(in_list):
return [normalize(name) for name in in_list]
_global_converter = _global_db = Database(website_timeout=(15, 30),
store_fetch_list=True)
def to_mapping(self) -> Mapping[str, Any]:
"""Convert the package recursively to a mapping s.t. it can be converted to JSON and similar formats."""
# yapf: disable
result = collections.OrderedDict([
("requirements",
collections.OrderedDict((key, req.to_mapping()) for key, req in sorted(self.requirements.items()))),
("rel_paths", sorted(str(rel_pth) for rel_pth in self.rel_paths)),
("unresolved_modules", [mod.to_mapping() for mod in self.unresolved_modules])
])
# yapf: enable
return result
_STDLIB_SET = set(stdlib_list.stdlib_list())
@icontract.require(lambda rel_paths: all(not rel_pth.is_absolute()
for rel_pth in rel_paths))
@icontract.ensure(lambda rel_paths, result: all(pth in result.rel_paths
for pth in rel_paths),
enabled=icontract.SLOW or READTHEDOCS,
description="Initial relative paths included")
@icontract.ensure(
lambda requirements, result: all(req.name in requirements
for req in result.requirements.values()),
enabled=icontract.SLOW or READTHEDOCS)
@icontract.require(
lambda requirements, module_to_requirement: missing_requirements(
module_to_requirement, requirements) == [],
import os
from collections import defaultdict
from pprint import pprint
import logging
import yaml
import sys
import copy
from stdlib_list import stdlib_list
import pkgutil
from fnmatch import fnmatch
import requests
logger = logging.getLogger('depfinder')
pyver = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
builtin_modules = stdlib_list(pyver)
del pyver
try:
# python 3
AST_TRY = [ast.Try]
except AttributeError:
# python 2.7
AST_TRY = [ast.TryExcept, ast.TryFinally]
# this AST_QUESTIONABLE list comprises the various ways an import can be weird
# 1. inside a try/except block
# 2. inside a function
# 3. inside a class
AST_QUESTIONABLE = tuple(
list(AST_TRY) + [ast.FunctionDef, ast.ClassDef, ast.If])
def get_stdlibs_list(python_version='3.5'):
return [l.split('.')[0] for l in stdlib_list(python_version)]
# ADDITIONAL TERMS are also included as allowed by Section 7 of the GNU
# Affero General Public License. These additional terms are Sections 1, 5,
# 6, 7, 8, and 9 from the Apache License, Version 2.0 (the "Apache-2.0")
# where all references to the definition "License" are instead defined to
# mean the AGPL-3.0+.
# You should have received a copy of the Apache-2.0 along with this
# program. If not, see <http://www.apache.org/licenses/LICENSE-2.0>.
import os
import ast
from stdlib_list import stdlib_list
STDLIB = set(stdlib_list())
IMPORT_TYPES = {
type(ast.parse('import george').body[0]), # type: ignore
type(ast.parse('import george as macdonald').body[0])} # type: ignore
IMPORT_FROM_TYPES = {
type(ast.parse('from george import macdonald').body[0]) # type: ignore
}
ALL_IMPORT_TYPES = IMPORT_TYPES.union(IMPORT_FROM_TYPES)
CONVERSIONS = {
'attr': 'attrs',
'PIL': 'Pillow',
from warehouse.packaging.interfaces import IFileStorage
from warehouse.packaging.models import (
Project, Release, Dependency, DependencyKind, Role, File, Filename,
JournalEntry, BlacklistedProject,
)
from warehouse.utils import http
MAX_FILESIZE = 60 * 1024 * 1024 # 60M
MAX_SIGSIZE = 8 * 1024 # 8K
PATH_HASHER = "blake2_256"
STDLIB_PROHIBITTED = {
packaging.utils.canonicalize_name(s.rstrip('-_.').lstrip('-_.'))
for s in chain.from_iterable(stdlib_list.stdlib_list(version)
for version in stdlib_list.short_versions)
}
# Wheel platform checking
# These platforms can be handled by a simple static list:
_allowed_platforms = {
"any",
"win32", "win_amd64", "win_ia64",
"manylinux1_x86_64", "manylinux1_i686",
"linux_armv6l", "linux_armv7l",
}
# macosx is a little more complicated:
_macosx_platform_re = re.compile("macosx_10_(\d+)+_(?P<arch>.*)")
_macosx_arches = {
"ppc", "ppc64",
#!/usr/bin/env python
import argparse
import logging
import os
import sys
import re
from os.path import join
from stdlib_list import stdlib_list
py2_version = "2.7"
py3_version = "3.4"
py2_libraries = stdlib_list(py2_version)
py3_libraries = stdlib_list(py3_version)
import_line_re = re.compile('^(\s+)?import\s|^(\s+)?from\s')
import_from_re = re.compile('^from\s+(?P<module_name>[A-z0-9_.]+)\s+import\s+(?P<imported_modules_names>[A-z0-9_., ]+)')
import_re = re.compile('^import\s+(?P<modules_names>[A-z0-9_., ]+)')
main_module_re = re.compile('^(?P<main_module>[A-z0-9_]+)\.')
modules = []
def module_in_the_line(line, module_list):
for module in module_list:
if line.startswith('import ' + module) or line.startswith('from ' + module):
return True
return False
from .model import Config, Integration
IGNORE_PACKAGES = {
commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS
}
PACKAGE_REGEX = re.compile(r"^(?:--.+\s)?([-_\.\w\d]+).*==.+$")
PIP_REGEX = re.compile(r"^(--.+\s)?([-_\.\w\d]+.*(?:==|>=|<=|~=|!=|<|>|===)?.*$)")
SUPPORTED_PYTHON_TUPLES = [
REQUIRED_PYTHON_VER[:2],
tuple(map(operator.add, REQUIRED_PYTHON_VER, (0, 1, 0)))[:2],
]
SUPPORTED_PYTHON_VERSIONS = [
".".join(map(str, version_tuple)) for version_tuple in SUPPORTED_PYTHON_TUPLES
]
STD_LIBS = {version: set(stdlib_list(version)) for version in SUPPORTED_PYTHON_VERSIONS}
PIPDEPTREE_CACHE = None
IGNORE_VIOLATIONS = {
# Still has standard library requirements.
"acmeda",
"blink",
"ezviz",
"hdmi_cec",
"juicenet",
"lupusec",
"rainbird",
"slide",
"suez_water",
}
MAX_SIGSIZE = 8 * 1024 # 8K
PATH_HASHER = "blake2_256"
def namespace_stdlib_list(module_list):
for module_name in module_list:
parts = module_name.split(".")
for i, part in enumerate(parts):
yield ".".join(parts[: i + 1])
STDLIB_PROHIBITTED = {
packaging.utils.canonicalize_name(s.rstrip("-_.").lstrip("-_."))
for s in chain.from_iterable(
namespace_stdlib_list(stdlib_list.stdlib_list(version))
for version in stdlib_list.short_versions
)
}
# Wheel platform checking
# Note: defining new platform ABI compatibility tags that don't
# have a python.org binary release to anchor them is a
# complex task that needs more than just OS+architecture info.
# For Linux specifically, the platform ABI is defined by each
# individual distro version, so wheels built on one version may
# not even work on older versions of the same distro, let alone
# a completely different distro.
#
# That means new entries should only be added given an
import voice_activity
from voice_activity.abc import (
AbstractCommand,
AbstractListener,
AbstractPlugin,
)
USER_CTX_TIMEOUT = 1200 # in seconds
COMMAND_REGEX = re.compile(r'([^"].*?|".*?")(?:\s|$)')
LOGGER = logging.getLogger(__name__)
PYTHON_VERSION = f"{sys.version_info[0]}.{sys.version_info[1]}"
EXCLUDE_FROM_AUTODISCOVERY = {"discord"} | set(dir(__builtins__)) | set(
sys.builtin_module_names) | set(stdlib_list(PYTHON_VERSION))
class VoiceActivity(discord.Client):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._users_context = {}
self._bot_commands = {}
self._bot_listeners = []
self._registered_plugins = set()
def autodiscover_plugins(self, mod, *args, visited_modules=None, **kwargs):
if not inspect.ismodule(mod):
raise ValueError(
f"Autodiscovery can only be done on modules. Passed {mode}")
keys = [
# noqa
import sys
import datetime
try:
from stdlib_list import stdlib_list
# import_standard_library
version = sys.version[:3]
libs = stdlib_list(version)
for lib in libs:
statement = 'import {}'.format(lib)
try:
exec(statement)
except ImportError:
pass
except:
pass
# stdlib extra imports
from collections import *
from functools import *
from unicodedata import *
from pprint import pprint
pp = pprint
File,
Filename,
JournalEntry,
BlacklistedProject,
)
from warehouse.utils import http
MAX_FILESIZE = 60 * 1024 * 1024 # 60M
MAX_SIGSIZE = 8 * 1024 # 8K
PATH_HASHER = "blake2_256"
STDLIB_PROHIBITTED = {
packaging.utils.canonicalize_name(s.rstrip('-_.').lstrip('-_.'))
for s in chain.from_iterable(
stdlib_list.stdlib_list(version)
for version in stdlib_list.short_versions)
}
# Wheel platform checking
# These platforms can be handled by a simple static list:
_allowed_platforms = {
"any",
"win32",
"win_amd64",
"win_ia64",
"manylinux1_x86_64",
"manylinux1_i686",
"linux_armv6l",
"linux_armv7l",
}
"""
Created on July 31, 2018
@author: Alejandro Molina
"""
import glob
import os
import re
import subprocess
from stdlib_list import stdlib_list
stdlibs = stdlib_list()
def is_std_lib(module_name):
return module_name in stdlibs
def is_module(name):
status, _ = subprocess.getstatusoutput("pip3 show " + name)
return status == 0
if __name__ == "__main__":
current_path = os.path.split(__file__)[0]
project_path = os.path.abspath(current_path + "/../")
print(project_path)
all_imports = set()
for filename in glob.iglob(project_path + "/**/*.py", recursive=True):
This file is part of OpenSesame.
OpenSesame is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenSesame is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OpenSesame. If not, see <http://www.gnu.org/licenses/>.
"""
from stdlib_list import stdlib_list
import sys
python_version = "%d.%d" % (sys.version_info[0], sys.version_info[1])
print(u'Python version: %s' % python_version)
fd = open(u'pkg-list-py%s.txt' % python_version, u'w')
for pkg in stdlib_list(python_version):
try:
__import__(pkg)
print('Imported %s' % pkg)
fd.write(pkg + u'\n')
except ImportError:
print('Failed to import %s' % pkg)
fd.close()
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function, division, absolute_import
import ast
import os
from collections import defaultdict
from stdlib_list import stdlib_list
import sys
pyver = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
builtin_modules = stdlib_list(pyver)
del pyver
del sys
class ImportCatcher(ast.NodeVisitor):
"""Find all imports in an Abstract Syntax Tree (AST).
Attributes
----------
required_modules : list
The list of imports that were found outside of try/except blocks
sketchy_modules : list
The list of imports that were found inside of try/except blocks
imports : list
The list of all ast.Import nodes in the AST
import_froms : list
import sys
import token
from collections import OrderedDict
from io import BytesIO
from pathlib import Path
from tokenize import tokenize, TokenInfo
from types import ModuleType
from typing import Sequence, Set
from stdlib_list import stdlib_list
from .exceptions import ValidationError
from .utils import pip_installable
py_version = ".".join(map(str, sys.version_info[:3]))
std_libs = stdlib_list(".".join(map(str, sys.version_info[:2])))
class ImportFinder(ast.NodeVisitor):
"""An AST node visitor to collect information about import statements.
Top-level package names like ``foo`` from import statements like
``import foo.bar``, ``from foo import bar as foobar`` etc. are
collected into the instance variable ``imported_names``, a set
of strings.
"""
def __init__(self, *args, **kwargs):
self.imported_names = set()
super(*args, **kwargs)
def analyse_code(self, code: str) -> Set[str]:
