import os from pysource.graph.installer import FailedToInstall from pysource.graph.space import is_builtin_module, FileExecutionError, extract_path_and_type from pysource.log import get_logger from pysource.api import get_import_names logger = get_logger("pysource.graph.model") class Package(object): """ A package is a top-level source folder. It contains several submodules. """ pymodule = None # :: PyModule rootpath = None # :: str submodules = None # :: [Module] dependencies = None # :: [str] def __init__(self, space, name): self.space = space # :: Space self.name = name # :: str def import_toplevel(self): """ import this package. """ self.pymodule = self.space.find_package(self.name) if not is_builtin_module(self.pymodule): self.rootpath = self.space.path_of_module(self.pymodule) logger.info("%s at %s", self.name, self.rootpath)
from pysource.log import get_logger from pysource.graph.cache import known_failures, shall_fail logger = get_logger('pysource.graph.installer') class FailedToInstall(Exception): def __init__(self, reason, package_name): super(FailedToInstall, self).__init__(reason, package_name) self.reason = reason self.package_name = package_name def __repr__(self): return 'FailedToInstall: installation of %s failed. reason: %s' % ( self.package_name, self.reason) def install_package(package_name): pip_install(package_name) def uninstall_package(package_name): pip_uninstall(package_name) # Internal: pip hacking. from pip.basecommand import load_all_commands, command_dict from pip.exceptions import InstallationError def initialize_pip_commands(): load_all_commands() return command_dict['install'], command_dict['uninstall'] pip_installer, pip_uninstaller = initialize_pip_commands()
""" Non-incremental graph builder. """ from collections import deque from pysource.graph.model import Package from pysource.graph.space import FileExecutionError from pysource.log import get_logger logger = get_logger('pysource.graph.builder') class GraphBuilder(object): def __init__(self, space, root_package_name, max_num_package): self.space = space if isinstance(root_package_name, list): self.root_packages = [ Package(self.space, name) for name in root_package_name ] else: self.root_packages = [Package(self.space, root_package_name)] self.max_num_package = max_num_package def build(self): packages_to_be_resolved = deque(self.root_packages) self.finished_packages = {} self.failed_packages = {} while (packages_to_be_resolved and len(self.finished_packages) < self.max_num_package): package = packages_to_be_resolved.popleft()
""" Non-incremental graph builder. """ from collections import deque from pysource.graph.model import Package from pysource.graph.space import FileExecutionError from pysource.log import get_logger logger = get_logger('pysource.graph.builder') class GraphBuilder(object): def __init__(self, space, root_package_name, max_num_package): self.space = space if isinstance(root_package_name, list): self.root_packages = [Package(self.space, name) for name in root_package_name] else: self.root_packages = [Package(self.space, root_package_name)] self.max_num_package = max_num_package def build(self): packages_to_be_resolved = deque(self.root_packages) self.finished_packages = {} self.failed_packages = {} while (packages_to_be_resolved and len(self.finished_packages) < self.max_num_package): package = packages_to_be_resolved.popleft() try:
import json from pysource.log import get_logger from pysource.graph.space import Space from pysource.graph.builder import GraphBuilder def dump_graph(graph, f): clean_graph = dict((pkg.name, pkg.dependencies) for pkg in graph.itervalues()) pickle.dump(clean_graph, f) def load_graph(f): return pickle.load(f) logger = get_logger("main-script") INPUT_FILE = "modules.json" with open(INPUT_FILE) as f: modules = json.load(f).keys() modules = modules[:50] logger.warn("modules=%s", modules) OUTPUT_FILE = "modules-graph.p" MAX_NUM_PACKAGES = 200 logger.warn("max_num_packages=%d", MAX_NUM_PACKAGES) builder = GraphBuilder(Space(), modules, max_num_package=MAX_NUM_PACKAGES)
from pysource.log import get_logger from pysource.graph.space import Space from pysource.graph.builder import GraphBuilder def dump_graph(graph, f): clean_graph = dict( (pkg.name, pkg.dependencies) for pkg in graph.itervalues()) pickle.dump(clean_graph, f) def load_graph(f): return pickle.load(f) logger = get_logger('main-script') INPUT_FILE = 'modules.json' with open(INPUT_FILE) as f: modules = json.load(f).keys() modules = modules[:50] logger.warn('modules=%s', modules) OUTPUT_FILE = 'modules-graph.p' MAX_NUM_PACKAGES = 200 logger.warn('max_num_packages=%d', MAX_NUM_PACKAGES) builder = GraphBuilder(Space(), modules, max_num_package=MAX_NUM_PACKAGES)
from pysource.log import get_logger from pysource.graph.cache import known_failures, shall_fail logger = get_logger("pysource.graph.installer") class FailedToInstall(Exception): def __init__(self, reason, package_name): super(FailedToInstall, self).__init__(reason, package_name) self.reason = reason self.package_name = package_name def __repr__(self): return "FailedToInstall: installation of %s failed. reason: %s" % (self.package_name, self.reason) def install_package(package_name): pip_install(package_name) def uninstall_package(package_name): pip_uninstall(package_name) # Internal: pip hacking. from pip.basecommand import load_all_commands, command_dict from pip.exceptions import InstallationError def initialize_pip_commands(): load_all_commands()
import os import sys from collections import defaultdict from pysource.log import get_logger logger = get_logger('pysource.graph.space') from pysource.graph.installer import (install_package, uninstall_package, FailedToInstall) # File was found, but it throws an error when it's imported.. class FileExecutionError(Exception): pass class Space(object): def __init__(self): self.packages = {} # name -> package_node self.files = {} # path -> package_node self.deferred_deps = [] # [(import_name, importer_path)] self.failures = defaultdict(lambda: []) def find_package(self, name): try: return find_module_under(name) except ImportError: # Package not found: installed it try: install_package(name) except FailedToInstall: raise ImportError return find_module_under(name)
import os import sys from collections import defaultdict from pysource.log import get_logger logger = get_logger('pysource.graph.space') from pysource.graph.installer import (install_package, uninstall_package, FailedToInstall) # File was found, but it throws an error when it's imported.. class FileExecutionError(Exception): pass class Space(object): def __init__(self): self.packages = {} # name -> package_node self.files = {} # path -> package_node self.deferred_deps = [] # [(import_name, importer_path)] self.failures = defaultdict(lambda: []) def find_package(self, name): try: return find_module_under(name) except ImportError: # Package not found: installed it try: install_package(name) except FailedToInstall: