def tags(cls): from packaging import tags for tag in tags.sys_tags(): yield tag # FIXME: Work around for https://github.com/pypa/packaging/pull/319 and Big Sur if sys.platform == 'darwin' and Version.from_string(platform.mac_ver()[0]) > Version(10): for override in tags.mac_platforms(version=(10, 16)): for tag in tags.sys_tags(): if not tag.platform: pass yield tags.Tag(tag.interpreter, tag.abi, override)
def is_cached(self): manifest = AutoInstall.manifest.get(self.name) if not manifest: return False if AutoInstall.overwrite_foreign_packages and manifest.get('index') != AutoInstall.index: return False if not manifest.get('version'): return False return not self.version or Version(*manifest.get('version').split('.')) in self.version
def is_cached(self): manifest = AutoInstall.manifest.get(self.name) if not manifest: return False if manifest.get('index') != AutoInstall.index: return False if not manifest.get('version'): return False return not self.version or Version( *manifest.get('version').split('.')) in self.version
class AutoInstall(object): DISABLE_ENV_VAR = 'DISABLE_WEBKITCOREPY_AUTOINSTALLER' directory = None index = 'pypi.org' timeout = 30 version = Version(sys.version_info[0], sys.version_info[1], sys.version_info[2]) packages = defaultdict(list) manifest = {} # Rely on our own certificates for PyPi, since we use PyPi to standardize root certificates ca_cert_path = os.path.join(os.path.dirname(__file__), 'cacert.pem') _previous_index = None _previous_ca_cert_path = None _fatal_check = False # When sharing an install location, projects may wish to overwrite packages on disk # originating from a different index. overwrite_foreign_packages = False @classmethod def _request(cls, url, ca_cert_path=None): context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) if ca_cert_path or cls.ca_cert_path: context.load_verify_locations( cafile=ca_cert_path or cls.ca_cert_path) return urlopen(url, timeout=cls.timeout, context=context) @classmethod def enabled(cls): if os.environ.get(cls.DISABLE_ENV_VAR) not in [ '0', 'FALSE', 'False', 'false', 'NO', 'No', 'no', None ]: return False return True if cls.directory else None @classmethod def userspace_should_own(cls, path): # Windows doesn't have sudo if not hasattr(os, "geteuid"): return # If we aren't root, the default behavior is correct if os.geteuid() != 0: return # If running as sudo, we really want the caller of sudo to own the autoinstall directory uid = int(os.environ.get('SUDO_UID', -1)) gid = int(os.environ.get('SUDO_GID', -1)) os.chown(path, uid, gid) if not os.path.isdir(path): return for root, directories, files in os.walk(path): for directory in directories: os.chown(os.path.join(root, directory), uid, gid) for file in files: os.chown(os.path.join(root, file), uid, gid) @classmethod def set_directory(cls, directory): if not directory or not isinstance(directory, str): raise ValueError( '{} is an invalid autoinstall directory'.format(directory)) if cls.enabled() is False: AutoInstall.log( 'Request to set autoinstall directory to {}'.format(directory)) AutoInstall.log( 'Environment variable {}={} overriding request'.format( cls.DISABLE_ENV_VAR, os.environ.get(cls.DISABLE_ENV_VAR), )) return directory = os.path.abspath(directory) if not os.path.isdir(directory): creation_root = directory while not os.path.isdir(os.path.dirname(creation_root)): creation_root = os.path.dirname(creation_root) if os.path.exists(directory): raise ValueError( '{} is not a directory and cannot be used as the autoinstall location' ) os.makedirs(directory) cls.userspace_should_own(creation_root) try: with open(os.path.join(directory, 'manifest.json'), 'r') as file: cls.manifest = json.load(file) except (IOError, OSError, ValueError): pass sys.path.insert(0, directory) cls.directory = directory @classmethod def _verify_index(cls): if not cls._previous_index: return def error(message): if cls._fatal_check: raise ValueError(message) sys.stderr.write('{}\n'.format(message)) sys.stderr.write('Falling back to previous index, {}\n\n'.format( cls._previous_index)) cls.index = cls._previous_index cls.ca_cert_path = cls._previous_ca_cert_path response = None try: response = AutoInstall._request('https://{}/simple/pip/'.format( cls.index), ca_cert_path=cls.ca_cert_path) if response.code != 200: error( 'Failed to set AutoInstall index to {}, received {} response when searching for simple/pip' .format(index, response.code)) except URLError: error( 'Failed to set AutoInstall index to {}, no response from the server' .format(cls.index)) finally: if response: response.close() cls._previous_index = None cls._previous_ca_cert_path = None cls._fatal_check = False @classmethod def set_index(cls, index, check=False, fatal=False, ca_cert_path=None): cls._previous_index = cls.index cls._previous_ca_cert_path = cls.ca_cert_path cls._fatal_check = fatal cls.index = index cls.ca_cert_path = ca_cert_path if check: cls._verify_index() return cls.index @classmethod def set_timeout(cls, timeout): if timeout is not None and timeout <= 0: raise ValueError('{} is an invalid timeout value'.format(timeout)) cls.timeout = math.ceil(timeout) @classmethod def register(cls, package, local=False): if isinstance(package, str): if cls.packages.get(package): return cls.packages[package] else: package = Package(package) elif isinstance(package, Package): if cls.packages.get(package.name): if cls.packages.get( package.name)[0].version != package.version: raise ValueError( 'Registered version of {} uses {}, but requested version uses {}' .format(package.name, cls.packages.get(package.name)[0].version, package.version)) return cls.packages.get(package.name) else: raise ValueError( 'Expected package to be str or Package, not {}'.format( type(package))) # If inside the WebKit checkout, a local library is likely checked in at Tools/Scripts/libraries. # When we detect such a library, we should not register it to be auto-installed if local and package.name != 'autoinstalled': libraries = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) checkout_root = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(libraries)))) for candidate in [ os.path.join(libraries, package.pypi_name), os.path.join(checkout_root, 'Internal', 'Tools', 'Scripts', 'libraries', package.pypi_name), ]: if candidate in sys.path: return package if not os.path.isdir(os.path.join(candidate, package.name)): continue sys.path.insert(0, candidate) return [package] for alias in package.aliases: cls.packages[alias].append(package) cls.packages[package.name].append(package) return [package] @classmethod def install(cls, package): return all( [to_install.install() for to_install in cls.register(package)]) @classmethod def install_everything(cls): for packages in cls.packages.values(): for package in packages: package.install() return None @classmethod def find_module(cls, fullname, path=None): if not cls.enabled() or path is not None: return name = fullname.split('.')[0] if cls.packages.get(name): cls.install(name) @classmethod def tags(cls): from packaging import tags for tag in tags.sys_tags(): yield tag # FIXME: Work around for https://github.com/pypa/packaging/pull/319 and Big Sur if sys.platform == 'darwin' and Version.from_string( platform.mac_ver()[0]) > Version(10): for override in tags.mac_platforms(version=(10, 16)): for tag in tags.sys_tags(): if not tag.platform: pass yield tags.Tag(tag.interpreter, tag.abi, override) @classmethod def log(cls, message, level=logging.WARNING): if not log.handlers or all( [isinstance(handle, NullHandler) for handle in log.handlers]): sys.stderr.write(message + '\n') else: log.log(level, message)
def archives(self): if self._archives: return self._archives AutoInstall._verify_index() path = 'simple/{}/'.format(self.pypi_name) response = AutoInstall._request('https://{}/{}'.format( AutoInstall.index, path)) try: if response.code != 200: raise ValueError('The package {} was not found on {}'.format( self.pypi_name, AutoInstall.index)) page = minidom.parseString(response.read()) cached_tags = None for element in reversed(page.getElementsByTagName("a")): if not len(element.childNodes): continue if element.childNodes[0].nodeType != minidom.Node.TEXT_NODE: continue attributes = {} for index in range(element.attributes.length): attributes[element.attributes.item( index).name] = element.attributes.item(index).value if not attributes.get('href', None): continue if self.wheel: match = re.search(r'.+-([^-]+-[^-]+-[^-]+).whl', element.childNodes[0].data) if not match: continue from packaging import tags if not cached_tags: cached_tags = set(AutoInstall.tags()) if all([ tag not in cached_tags for tag in tags.parse_tag(match.group(1)) ]): continue extension = 'whl' else: if element.childNodes[0].data.endswith( ('.tar.gz', '.tar.bz2')): extension = 'tar.gz' elif element.childNodes[0].data.endswith('.zip'): extension = 'zip' else: continue requires = attributes.get('data-requires-python') if requires and not AutoInstall.version.matches(requires): continue version_candidate = re.search(r'\d+\.\d+(\.\d+)?', element.childNodes[0].data) if not version_candidate: continue version = Version(*version_candidate.group().split('.')) if self.version and version not in self.version: continue link = attributes['href'].split('#')[0] if '://' not in link: depth = 0 while link.startswith('../'): depth += 1 link = link[3:] link = 'https://{}/{}{}'.format( AutoInstall.index, '/'.join(path.split('/')[depth:]), link) self._archives.append( self.Archive( name=self.pypi_name, link=link, version=version, extension=extension, )) self._archives = sorted(self._archives, key=lambda archive: archive.version) return self._archives finally: response.close()
import sys log = logging.getLogger('webkitscmpy') def _maybe_add_webkitcorepy_path(): # Hopefully we're beside webkitcorepy, otherwise webkitcorepy will need to be installed. libraries_path = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) webkitcorepy_path = os.path.join(libraries_path, 'webkitcorepy') if os.path.isdir(webkitcorepy_path) and os.path.isdir(os.path.join(webkitcorepy_path, 'webkitcorepy')) and webkitcorepy_path not in sys.path: sys.path.insert(0, webkitcorepy_path) _maybe_add_webkitcorepy_path() try: from webkitcorepy.version import Version except ImportError: raise ImportError( "'webkitcorepy' could not be found on your Python path.\n" + "You are not running from a WebKit checkout.\n" + "Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`" ) version = Version(0, 0, 3) from webkitscmpy import local from webkitscmpy import mocks name = 'webkitscmpy'
import sys from logging import NullHandler log = logging.getLogger('webkitcorepy') log.addHandler(NullHandler()) from webkitcorepy.version import Version from webkitcorepy.string_utils import BytesIO, StringIO, UnicodeIO, unicode from webkitcorepy.timeout import Timeout from webkitcorepy.subprocess_utils import TimeoutExpired, CompletedProcess, run from webkitcorepy.output_capture import LoggerCapture, OutputCapture, OutputDuplicate from webkitcorepy.task_pool import TaskPool from webkitcorepy.credentials import credentials version = Version(0, 5, 5) from webkitcorepy.autoinstall import Package, AutoInstall if sys.version_info > (3, 0): AutoInstall.register(Package('mock', Version(4))) else: AutoInstall.register(Package('mock', Version(3, 0, 5))) if platform.system() == 'Windows': AutoInstall.register( Package('win_inet_pton', Version(1, 1, 0), pypi_name='win-inet-pton')) AutoInstall.register(Package('certifi', Version(2020, 6, 20))) AutoInstall.register(Package('chardet', Version(3, 0, 4))) AutoInstall.register(
import platform import sys from logging import NullHandler log = logging.getLogger('webkitcorepy') log.addHandler(NullHandler()) from webkitcorepy.version import Version from webkitcorepy.string_utils import BytesIO, StringIO, UnicodeIO, unicode from webkitcorepy.timeout import Timeout from webkitcorepy.subprocess_utils import TimeoutExpired, CompletedProcess, run from webkitcorepy.output_capture import LoggerCapture, OutputCapture, OutputDuplicate from webkitcorepy.task_pool import TaskPool version = Version(0, 5, 0) from webkitcorepy.autoinstall import Package, AutoInstall if sys.version_info > (3, 0): AutoInstall.register(Package('mock', Version(4))) else: AutoInstall.register(Package('mock', Version(3, 0, 5))) if platform.system() == 'Windows': AutoInstall.register( Package('win_inet_pton', Version(1, 1, 0), pypi_name='win-inet-pton')) AutoInstall.register(Package('certifi', Version(2020, 6, 20))) AutoInstall.register(Package('chardet', Version(3, 0, 4))) AutoInstall.register(Package('funcsigs', Version(1, 0, 2)))
import os import sys def _maybe_add_webkit_python_library_paths(): # Hopefully we're beside webkit*py libraries, otherwise webkit*py will need to be installed. libraries_path = os.path.dirname( os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) for library in ['webkitcorepy', 'webkitflaskpy']: library_path = os.path.join(libraries_path, library) if os.path.isdir(library_path) and os.path.isdir( os.path.join(library_path, library)) and library_path not in sys.path: sys.path.insert(0, library_path) _maybe_add_webkit_python_library_paths() try: from webkitcorepy.version import Version except ImportError: raise ImportError( "'webkitcorepy' could not be found on your Python path.\n" + "You are not running from a WebKit checkout.\n" + "Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`" ) version = Version(1, 1, 0) name = 'resultsdbpy'
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os import sys def _maybe_add_webkitcorepy_path(): # Hopefully we're beside webkitcorepy, otherwise webkitcorepy will need to be installed. libraries_path = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) webkitcorepy_path = os.path.join(libraries_path, 'webkitcorepy') if os.path.isdir(webkitcorepy_path) and os.path.isdir(os.path.join(webkitcorepy_path, 'webkitcorepy')) and webkitcorepy_path not in sys.path: sys.path.insert(0, webkitcorepy_path) _maybe_add_webkitcorepy_path() try: from webkitcorepy.version import Version except ImportError: raise ImportError( "'webkitcorepy' could not be found on your Python path.\n" + "You are not running from a WebKit checkout.\n" + "Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`" ) version = Version(1, 0, 1) name = 'resultsdbpy'
class AutoInstall(object): _enabled = None enabled = False directory = None index = 'pypi.org' timeout = 30 version = Version(sys.version_info[0], sys.version_info[1], sys.version_info[2]) packages = {} manifest = {} @classmethod def _request(cls, url): # Rely on our own certificates for PyPi, since we use PyPi to standardize root certificates if url.startswith('https://pypi.org') or url.startswith( 'https://files.pythonhosted.org'): return urlopen( url, timeout=cls.timeout, cafile=os.path.join(os.path.dirname(__file__), 'cacert.pem'), ) return urlopen(url, timeout=cls.timeout) @classmethod def enable(cls): cls._enabled = True cls.enabled = True @classmethod def disable(cls): cls._enabled = False cls.enabled = False @classmethod def userspace_should_own(cls, path): # Windows doesn't have sudo if not hasattr(os, "geteuid"): return # If we aren't root, the default behavior is correct if os.geteuid() != 0: return # If running as sudo, we really want the caller of sudo to own the autoinstall directory uid = int(os.environ.get('SUDO_UID', -1)) gid = int(os.environ.get('SUDO_GID', -1)) os.chown(path, uid, gid) if not os.path.isdir(path): return for root, directories, files in os.walk(path): for directory in directories: os.chown(os.path.join(root, directory), uid, gid) for file in files: os.chown(os.path.join(root, file), uid, gid) @classmethod def set_directory(cls, directory): if not directory or not isinstance(directory, str): raise ValueError( '{} is an invalid autoinstall directory'.format(directory)) directory = os.path.abspath(directory) if not os.path.isdir(directory): creation_root = directory while not os.path.isdir(os.path.dirname(creation_root)): creation_root = os.path.dirname(creation_root) if os.path.exists(directory): raise ValueError( '{} is not a directory and cannot be used as the autoinstall location' ) os.makedirs(directory) cls.userspace_should_own(creation_root) try: with open(os.path.join(directory, 'manifest.json'), 'r') as file: cls.manifest = json.load(file) except (IOError, OSError, ValueError): pass sys.path.insert(0, directory) cls.directory = directory if cls._enabled is None: cls._enabled = True cls.enabled = True @classmethod def set_index(cls, index, check=True): if check: response = AutoInstall._request( 'https://{}/simple/pip/'.format(index)) try: if response.code != 200: raise ValueError( 'Failed to set AutoInstall index to {}, received {} response when searching for pip' .format(index, response.code)) finally: response.close() cls.index = index @classmethod def set_timeout(cls, timeout): if timeout is not None and timeout <= 0: raise ValueError('{} is an invalid timeout value'.format(timeout)) cls.timeout = math.ceil(timeout) @classmethod def register(cls, package): if isinstance(package, str): if cls.packages.get(package): return cls.packages[package] else: package = Package(package) elif isinstance(package, Package): if cls.packages.get(package.name): if cls.packages.get(package.name).version != package.version: raise ValueError( 'Registered version of {} uses {}, but requested version uses {}' .format(package.name, cls.packages.get(package.name).version, package.version)) return cls.packages.get(package.name) else: raise ValueError( 'Expected package to be str or Package, not {}'.format( type(package))) for alias in package.aliases: cls.packages[alias] = package cls.packages[package.name] = package return package @classmethod def install(cls, package): to_install = cls.register(package) return to_install.install() @classmethod def install_everything(cls): for package in cls.packages.values(): package.install() return None @classmethod def find_module(cls, fullname, path=None): if not cls.enabled or path is not None: return name = fullname.split('.')[0] if cls.packages.get(name): cls.install(name) @classmethod def tags(cls): from packaging import tags for tag in tags.sys_tags(): yield tag # FIXME: Work around for https://github.com/pypa/packaging/pull/319 and Big Sur if sys.platform == 'darwin' and Version.from_string( platform.mac_ver()[0]) > Version(10): for override in tags.mac_platforms(version=(10, 16)): for tag in tags.sys_tags(): if not tag.platform: pass yield tags.Tag(tag.interpreter, tag.abi, override)
import os import sys def _maybe_add_webkit_python_library_paths(): # Hopefully we're beside webkit*py libraries, otherwise webkit*py will need to be installed. libraries_path = os.path.dirname( os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) for library in ['webkitcorepy', 'webkitscmpy', 'webkitflaskpy']: library_path = os.path.join(libraries_path, library) if os.path.isdir(library_path) and os.path.isdir( os.path.join(library_path, library)) and library_path not in sys.path: sys.path.insert(0, library_path) _maybe_add_webkit_python_library_paths() try: from webkitcorepy.version import Version except ImportError: raise ImportError( "'webkitcorepy' could not be found on your Python path.\n" + "You are not running from a WebKit checkout.\n" + "Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`" ) version = Version(2, 0, 3) name = 'resultsdbpy'
def archives(self): if self._archives: return self._archives AutoInstall._verify_index() path = 'simple/{}/'.format(self.pypi_name) response = AutoInstall._request('https://{}/{}'.format( AutoInstall.index, path)) try: if response.code != 200: raise ValueError('The package {} was not found on {}'.format( self.pypi_name, AutoInstall.index)) packages = SimplyPypiIndexPageParser.parse( response.read().decode("UTF-8")) cached_tags = None for package in reversed(packages): if self.wheel: match = re.search(r'.+-([^-]+-[^-]+-[^-]+).whl', package['name']) if not match: continue from packaging import tags if not cached_tags: cached_tags = set(AutoInstall.tags()) if all([ tag not in cached_tags for tag in tags.parse_tag(match.group(1)) ]): continue extension = 'whl' else: if package['name'].endswith(('.tar.gz', '.tar.bz2')): extension = 'tar.gz' elif package['name'].endswith('.zip'): extension = 'zip' else: continue requires = package.get('data-requires-python') if requires and not AutoInstall.version.matches(requires): continue version_candidate = re.search(r'\d+\.\d+(\.\d+)?', package["name"]) if not version_candidate: continue version = Version(*version_candidate.group().split('.')) if self.version and version not in self.version: continue link = package['href'].split('#')[0] if '://' not in link: depth = 0 while link.startswith('../'): depth += 1 link = link[3:] link = 'https://{}/{}{}'.format( AutoInstall.index, '/'.join(path.split('/')[depth:]), link) self._archives.append( self.Archive( name=self.pypi_name, link=link, version=version, extension=extension, )) self._archives = sorted(self._archives, key=lambda archive: archive.version) return self._archives finally: response.close()
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import logging import platform import sys log = logging.getLogger('webkitcorepy') from webkitcorepy.version import Version from webkitcorepy.string_utils import BytesIO, StringIO, UnicodeIO, unicode version = Version(0, 1, 1) from webkitcorepy.autoinstall import Package, AutoInstall if sys.version_info > (3, 0): AutoInstall.register(Package('mock', Version(4))) AutoInstall.register(Package('setuptools', Version(41, 2, 0))) else: AutoInstall.register(Package('mock', Version(3, 0, 5))) AutoInstall.register(Package('setuptools', Version(41, 0, 1))) if platform.system() == 'Windows': AutoInstall.register(Package('win_inet_pton', Version(1, 1, 0), pypi_name='win-inet-pton')) AutoInstall.register(Package('certifi', Version(2020, 6, 20))) AutoInstall.register(Package('chardet', Version(3, 0, 4))) AutoInstall.register(Package('funcsigs', Version(1, 0, 2))) AutoInstall.register(Package('idna', Version(2, 10)))
import os import sys def _maybe_add_webkitcorepy_path(): # Hopefully we're beside webkitcorepy, otherwise webkitcorepy will need to be installed. libraries_path = os.path.dirname( os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) webkitcorepy_path = os.path.join(libraries_path, 'webkitcorepy') if os.path.isdir(webkitcorepy_path) and os.path.isdir( os.path.join( webkitcorepy_path, 'webkitcorepy')) and webkitcorepy_path not in sys.path: sys.path.insert(0, webkitcorepy_path) _maybe_add_webkitcorepy_path() try: from webkitcorepy.version import Version except ImportError: raise ImportError( "'webkitcorepy' could not be found on your Python path.\n" + "You are not running from a WebKit checkout.\n" + "Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`" ) version = Version(1, 0, 0) name = 'resultsdbpy'