import os
from pysource.graph.installer import FailedToInstall
from pysource.graph.space import is_builtin_module, FileExecutionError, extract_path_and_type
from pysource.log import get_logger
from pysource.api import get_import_names
logger = get_logger("pysource.graph.model")
class Package(object):
""" A package is a top-level source folder.
It contains several submodules.
"""
pymodule = None # :: PyModule
rootpath = None # :: str
submodules = None # :: [Module]
dependencies = None # :: [str]
def __init__(self, space, name):
self.space = space # :: Space
self.name = name # :: str
def import_toplevel(self):
""" import this package.
"""
self.pymodule = self.space.find_package(self.name)
if not is_builtin_module(self.pymodule):
self.rootpath = self.space.path_of_module(self.pymodule)
logger.info("%s at %s", self.name, self.rootpath)
from pysource.log import get_logger
from pysource.graph.cache import known_failures, shall_fail
logger = get_logger('pysource.graph.installer')
class FailedToInstall(Exception):
def __init__(self, reason, package_name):
super(FailedToInstall, self).__init__(reason, package_name)
self.reason = reason
self.package_name = package_name
def __repr__(self):
return 'FailedToInstall: installation of %s failed. reason: %s' % (
self.package_name, self.reason)
def install_package(package_name):
pip_install(package_name)
def uninstall_package(package_name):
pip_uninstall(package_name)
# Internal: pip hacking.
from pip.basecommand import load_all_commands, command_dict
from pip.exceptions import InstallationError
def initialize_pip_commands():
load_all_commands()
return command_dict['install'], command_dict['uninstall']
pip_installer, pip_uninstaller = initialize_pip_commands()
""" Non-incremental graph builder.
"""
from collections import deque
from pysource.graph.model import Package
from pysource.graph.space import FileExecutionError
from pysource.log import get_logger
logger = get_logger('pysource.graph.builder')
class GraphBuilder(object):
def __init__(self, space, root_package_name, max_num_package):
self.space = space
if isinstance(root_package_name, list):
self.root_packages = [
Package(self.space, name) for name in root_package_name
]
else:
self.root_packages = [Package(self.space, root_package_name)]
self.max_num_package = max_num_package
def build(self):
packages_to_be_resolved = deque(self.root_packages)
self.finished_packages = {}
self.failed_packages = {}
while (packages_to_be_resolved
and len(self.finished_packages) < self.max_num_package):
package = packages_to_be_resolved.popleft()
""" Non-incremental graph builder.
"""
from collections import deque
from pysource.graph.model import Package
from pysource.graph.space import FileExecutionError
from pysource.log import get_logger
logger = get_logger('pysource.graph.builder')
class GraphBuilder(object):
def __init__(self, space, root_package_name, max_num_package):
self.space = space
if isinstance(root_package_name, list):
self.root_packages = [Package(self.space, name)
for name in root_package_name]
else:
self.root_packages = [Package(self.space, root_package_name)]
self.max_num_package = max_num_package
def build(self):
packages_to_be_resolved = deque(self.root_packages)
self.finished_packages = {}
self.failed_packages = {}
while (packages_to_be_resolved and
len(self.finished_packages) < self.max_num_package):
package = packages_to_be_resolved.popleft()
try:
import json
from pysource.log import get_logger
from pysource.graph.space import Space
from pysource.graph.builder import GraphBuilder
def dump_graph(graph, f):
clean_graph = dict((pkg.name, pkg.dependencies) for pkg in graph.itervalues())
pickle.dump(clean_graph, f)
def load_graph(f):
return pickle.load(f)
logger = get_logger("main-script")
INPUT_FILE = "modules.json"
with open(INPUT_FILE) as f:
modules = json.load(f).keys()
modules = modules[:50]
logger.warn("modules=%s", modules)
OUTPUT_FILE = "modules-graph.p"
MAX_NUM_PACKAGES = 200
logger.warn("max_num_packages=%d", MAX_NUM_PACKAGES)
builder = GraphBuilder(Space(), modules, max_num_package=MAX_NUM_PACKAGES)
from pysource.log import get_logger
from pysource.graph.space import Space
from pysource.graph.builder import GraphBuilder
def dump_graph(graph, f):
clean_graph = dict(
(pkg.name, pkg.dependencies) for pkg in graph.itervalues())
pickle.dump(clean_graph, f)
def load_graph(f):
return pickle.load(f)
logger = get_logger('main-script')
INPUT_FILE = 'modules.json'
with open(INPUT_FILE) as f:
modules = json.load(f).keys()
modules = modules[:50]
logger.warn('modules=%s', modules)
OUTPUT_FILE = 'modules-graph.p'
MAX_NUM_PACKAGES = 200
logger.warn('max_num_packages=%d', MAX_NUM_PACKAGES)
builder = GraphBuilder(Space(), modules, max_num_package=MAX_NUM_PACKAGES)
from pysource.log import get_logger
from pysource.graph.cache import known_failures, shall_fail
logger = get_logger("pysource.graph.installer")
class FailedToInstall(Exception):
def __init__(self, reason, package_name):
super(FailedToInstall, self).__init__(reason, package_name)
self.reason = reason
self.package_name = package_name
def __repr__(self):
return "FailedToInstall: installation of %s failed. reason: %s" % (self.package_name, self.reason)
def install_package(package_name):
pip_install(package_name)
def uninstall_package(package_name):
pip_uninstall(package_name)
# Internal: pip hacking.
from pip.basecommand import load_all_commands, command_dict
from pip.exceptions import InstallationError
def initialize_pip_commands():
load_all_commands()
import os
import sys
from collections import defaultdict
from pysource.log import get_logger
logger = get_logger('pysource.graph.space')
from pysource.graph.installer import (install_package, uninstall_package,
FailedToInstall)
# File was found, but it throws an error when it's imported..
class FileExecutionError(Exception):
pass
class Space(object):
def __init__(self):
self.packages = {} # name -> package_node
self.files = {} # path -> package_node
self.deferred_deps = [] # [(import_name, importer_path)]
self.failures = defaultdict(lambda: [])
def find_package(self, name):
try:
return find_module_under(name)
except ImportError:
# Package not found: installed it
try:
install_package(name)
except FailedToInstall:
raise ImportError
return find_module_under(name)
import os
import sys
from collections import defaultdict
from pysource.log import get_logger
logger = get_logger('pysource.graph.space')
from pysource.graph.installer import (install_package, uninstall_package,
FailedToInstall)
# File was found, but it throws an error when it's imported..
class FileExecutionError(Exception):
pass
class Space(object):
def __init__(self):
self.packages = {} # name -> package_node
self.files = {} # path -> package_node
self.deferred_deps = [] # [(import_name, importer_path)]
self.failures = defaultdict(lambda: [])
def find_package(self, name):
try:
return find_module_under(name)
except ImportError:
# Package not found: installed it
try:
install_package(name)
except FailedToInstall:
