def test_optproblems_solution(self):
# test to make sure that at the specified solution point, the objective
# values match what is given in the solution
#find all the optproblems in lib
startdirs = [os.path.dirname(openmdao.lib.optproblems.__file__),]
psta = PythonSourceTreeAnalyser(startdirs, os.path.join('*','test','*'))
opt_problems = psta.find_inheritors("openmdao.main.problem_formulation.OptProblem")
for prob_name in opt_problems:
#print "running %s"%prob_name
prob_class = prob_name.split(".")[-1]
prob_package = ".".join(prob_name.split(".")[:-1])
prob_package = __import__(prob_package,globals(),locals(),[prob_class,],-1)
prob = getattr(prob_package,prob_class)() #create instance of the OptProblem
prob = set_as_top(prob)
try:
prob.check_solution(strict=True)
except ValueError as err:
self.fail("There is missing piece of the solution for %s%s"%(prob.__class__,str(err)))
prob.architecture = OptProblemSolutionCheck()
prob.run()
error = prob.check_solution(strict=True)
self.assertAccuracy(prob_name,error,.001)
def build_optproblem_list(include=[], exclude=[]):
"""builds a list of optproblems
include: (optional) list of optproblems names
The names of the optproblems to test. Only optproblems in this list
will be tested. Each name should be just the class name (e.g., 'SellarProblem').
Must be set to None, if excludes is specified. If not specified,
all OptProblems, except those in exclude are used.
exclude: (optional) list of optproblems names
The names of the optproblems not to test. All optproblems from
openmdao.lib.optproblems will be tested, except for the ones in this
list. Each name should just be the class name (e.g. 'SellarProblem').
Must be set to None, if includes is specified.
"""
if include and exclude:
raise ValueError("Can't set both include and exlude for OptProblems")
startdirs = [os.path.dirname(openmdao.lib.optproblems.__file__),
os.path.dirname(openmdao.main.__file__)]
psta = PythonSourceTreeAnalyser(startdirs, os.path.join('*','test','*'))
opt_problems = psta.find_inheritors("openmdao.main.problem_formulation.OptProblem")
probs = []
for prob_name in opt_problems:
prob_class = prob_name.split(".")[-1]
prob_package = ".".join(prob_name.split(".")[:-1])
if (not include and not exclude) or (include and prob_class in include) or \
(exclude and prob_class not in exclude):
prob_package = __import__(prob_package,globals(),locals(),[prob_class,],-1)
probs.append(getattr(prob_package,prob_class)()) #create instance of the OptProblem
return probs
def build_optproblem_list(include=[], exclude=[]):
"""builds a list of optproblems
include: (optional) list of optproblems names
the names of the optproblems to test. Only optproblems in this list
will be tested. Each name should just be the class name (e.g. 'SellarProblem').
Must be set to None, if excludes is specified. If not specified,
all OptProblems, except those in exclude are used.
exclude: (optional) list of optproblems names
the names of the optproblems not to test. All optproblems from
openmdao.lib.optproblems will be tested, except for the ones in this
list. Each name should just be the class name (e.g. 'SellarProblem').
Must be set to None, if includes is specified.
"""
if include and exclude:
raise ValueError("Can't set both include and exlude")
startdirs = [os.path.dirname(openmdao.lib.optproblems.__file__),
os.path.dirname(openmdao.main.__file__)]
psta = PythonSourceTreeAnalyser(startdirs, os.path.join('*','test','*'))
opt_problems = psta.find_inheritors("openmdao.main.problem_formulation.OptProblem")
probs = []
for prob_name in opt_problems:
prob_class = prob_name.split(".")[-1]
prob_package = ".".join(prob_name.split(".")[:-1])
if (not include and not exclude) or (include and prob_class in include) or \
(exclude and prob_class not in exclude):
prob_package = __import__(prob_package,globals(),locals(),[prob_class,],-1)
probs.append(getattr(prob_package,prob_class)()) #create instance of the OptProblem
return probs
def test_optproblems_solution(self):
#find all the optproblems in lib
startdirs = [
os.path.dirname(openmdao.lib.optproblems.__file__),
]
psta = PythonSourceTreeAnalyser(startdirs,
os.path.join('*', 'test', '*'))
opt_problems = psta.find_inheritors(
"openmdao.main.problem_formulation.OptProblem")
for prob_name in opt_problems:
#print "running %s"%prob_name
prob_class = prob_name.split(".")[-1]
prob_package = ".".join(prob_name.split(".")[:-1])
prob_package = __import__(prob_package, globals(), locals(), [
prob_class,
], -1)
prob = getattr(prob_package,
prob_class)() #create instance of the OptProblem
prob.architecture = OptProblemSolutionCheck()
prob.configure()
prob.run()
error = prob.check_solution(strict=True)
self.assertAccuracy(prob_name, error, .001)
def build_arch_list(include=[], exclude=[]):
"""builds a list of architectures
include: list of architecture names
the names of the architectures to test. Only architectures in this list
will be tested. Each name should just be the class name (e.g. 'MDF', 'CO').
Must be set to None, if excludes is specified
exclude: list of architecture names
the names of the architectures not to test. All architectures from
openmdao.lib.architectures will be tested, except for the ones in this
list. Each name should just be the class name (e.g. 'MDF', 'CO').
Must be set to None, if includes is specified
"""
if include and exclude:
raise ValueError("Can't set both include and exlude")
startdirs = [os.path.dirname(openmdao.lib.architectures.__file__),
os.path.dirname(openmdao.main.__file__)]
psta = PythonSourceTreeAnalyser(startdirs, os.path.join('*','test','*'))
architectures = psta.find_inheritors("openmdao.main.arch.Architecture")
archs = []
for arch_name in architectures:
arch_class = arch_name.split(".")[-1]
arch_package = ".".join(arch_name.split(".")[:-1])
if (not include and not exclude) or (include and arch_class in include) or \
(exclude and arch_class not in exclude):
arch_package = __import__(arch_package,globals(),locals(),[arch_class,],-1)
archs.append(getattr(arch_package,arch_class)()) #create instance of the Architecture
return archs
def __init__(self, groups=plugin_groups.keys(), search_path=None):
super(PkgResourcesFactory, self).__init__()
self._have_new_types = True
self._groups = copy.copy(groups)
self._search_path = search_path
self.env = Environment(search_path)
self.tree_analyser = PythonSourceTreeAnalyser()
def build_arch_list(include=[], exclude=[]):
"""Builds a list of architectures.
include: list of architecture names
The names of the architectures to test. Only architectures in this list
will be tested. Each name should be just the class name (e.g. 'MDF', 'CO').
Must be set to None, if excludes is specified
exclude: list of architecture names
The names of the architectures not to test. All architectures from
openmdao.lib.architectures will be tested, except for the ones in this
list. Each name should be just the class name (e.g., 'MDF', 'CO').
Must be set to None, if includes is specified.
"""
if include and exclude:
raise ValueError("Can't set both include and exlude")
startdirs = [os.path.dirname(openmdao.lib.architectures.__file__),
os.path.dirname(openmdao.main.__file__)]
psta = PythonSourceTreeAnalyser(startdirs, os.path.join('*','test','*'))
architectures = psta.find_inheritors("openmdao.main.arch.Architecture")
archs = []
for arch_name in architectures:
arch_class = arch_name.split(".")[-1]
arch_package = ".".join(arch_name.split(".")[:-1])
if (not include and not exclude) or (include and arch_class in include) or \
(exclude and arch_class not in exclude):
arch_package = __import__(arch_package,globals(),locals(),[arch_class,],-1)
archs.append(getattr(arch_package,arch_class)()) #create instance of the Architecture
return archs
def _find_all_plugins(searchdir):
"""Return a dict containing lists of each plugin type found, keyed by
plugin group name, e.g., openmdao.component, openmdao.variable, etc.
"""
dct = {}
modnames = ['openmdao.main',
'openmdao.lib.datatypes',
'openmdao.lib.components',
'openmdao.lib.drivers',
'openmdao.lib.surrogatemodels',
'openmdao.lib.doegenerators',
'openmdao.lib.differentiators',
'openmdao.lib.optproblems',
'openmdao.lib.casehandlers',
'openmdao.lib.architectures']
modules = []
for mod in modnames:
try:
__import__(mod)
except ImportError:
print 'skipping import of %s' % mod
else:
modules.append(sys.modules[mod])
dirs = [os.path.dirname(m.__file__) for m in modules]+[searchdir]
psta = PythonSourceTreeAnalyser(dirs, exclude=_exclude_funct)
for key, val in plugin_groups.items():
dct[key] = set(psta.find_inheritors(val))
return dct
def test_PythonSourceTreeAnalyser(self):
try:
import openmdao.main
import openmdao.lib
except ImportError:
# don't perform this test if openmdao.main
# and openmdao.lib aren't present
raise SkipTest("this test requires openmdao.main and openmdao.lib")
def exclude_tests(pname):
parts = pname.split(os.sep)
return 'test' in parts
startdirs = [os.path.dirname(openmdao.main.__file__),
os.path.dirname(openmdao.lib.__file__)]
psta = PythonSourceTreeAnalyser(startdirs, exclude_tests)
self.assertTrue('openmdao.main.component.Component' in
psta.graph['openmdao.main.container.Container'])
self.assertTrue('openmdao.main.assembly.Assembly' in
psta.graph['openmdao.main.component.Component'])
self.assertTrue('openmdao.main.datatypes.float.Float' in
psta.graph['openmdao.main.variable.Variable'])
comps = psta.find_inheritors('openmdao.main.component.Component')
icomps = psta.find_inheritors('IComponent')
self.assertTrue('openmdao.main.assembly.Assembly' in icomps)
comps.extend(psta.find_inheritors('openmdao.main.variable.Variable'))
comps.extend(psta.find_inheritors('enthought.traits.api.Array'))
comps = [x.rsplit('.',1)[1] for x in comps if '.examples.' not in x and '.optproblems.' not in x]
cset = set(comps)
excludes = set([
'Driver',
'DriverUsesDerivatives',
'DistributionCaseDriver',
'CaseIterDriverBase',
'PassthroughTrait',
'PassthroughProperty',
'OptProblem',
'TraitArray',
'Broadcast', # utility class for bliss2000
'SubSystemOpt', # utility class for bliss2000
'SubSystemObj' # utility class for bliss2000
])
cset = cset - excludes
from openmdao.main.api import get_available_types
types = set([x[0] for x in get_available_types()])
types = [x.rsplit('.',1)[1] for x in types if x.startswith('openmdao.')]
tset = set(types)
noentrypts = cset-tset
if noentrypts:
self.fail("the following Components are not registered using entry points: %s" % noentrypts)
def find_all_plugins(searchdir):
"""Return a dict containing lists of each plugin type found, keyed by
plugin group name, e.g., openmdao.component, openmdao.variable, etc.
"""
dct = {}
psta = PythonSourceTreeAnalyser(searchdir, exclude=_exclude_funct)
for key, lst in plugin_groups.items():
epset = set(psta.find_inheritors(lst[0]))
if epset:
dct[key] = epset
return dct
def find_all_plugins(searchdir):
"""Return a dict containing lists of each plugin type found, keyed by
plugin group name, e.g., openmdao.component, openmdao.variable, etc.
"""
dct = {}
psta = PythonSourceTreeAnalyser(searchdir, exclude=_exclude_funct)
for key, lst in plugin_groups.items():
epset = set(psta.find_inheritors(lst[0]))
if epset:
dct[key] = epset
return dct
def test_PythonSourceTreeAnalyser(self):
def exclude(pname):
keywords = set(["test", "docs", "examples", "optproblems"])
parts = set(pname.split(os.sep))
return keywords.intersection(parts)
psta = PythonSourceTreeAnalyser(self.startdirs, exclude)
self.assertTrue("openmdao.main.component.Component" in psta.graph["openmdao.main.container.Container"])
self.assertTrue("openmdao.main.assembly.Assembly" in psta.graph["openmdao.main.component.Component"])
self.assertTrue("openmdao.main.datatypes.float.Float" in psta.graph["openmdao.main.variable.Variable"])
comps = psta.find_inheritors("openmdao.main.component.Component")
icomps = psta.find_inheritors("IComponent")
self.assertTrue("openmdao.main.assembly.Assembly" in icomps)
comps.extend(psta.find_inheritors("openmdao.main.variable.Variable"))
comps.extend(psta.find_inheritors("enthought.traits.api.Array"))
comps = [x.rsplit(".", 1)[1] for x in comps]
# comps = [x.rsplit('.',1)[1] for x in comps if '.examples.' not in x and '.optproblems.' not in x]
cset = set(comps)
excludes = set(
[
"Driver",
"DriverUsesDerivatives",
"DistributionCaseDriver",
"CaseIterDriverBase",
"PassthroughTrait",
"PassthroughProperty",
"OptProblem",
"TraitArray",
"Broadcast", # utility class for bliss2000
"SubSystemOpt", # utility class for bliss2000
"SubSystemObj", # utility class for bliss2000
]
)
cset = cset - excludes
from openmdao.main.api import get_available_types
types = set([x[0] for x in get_available_types()])
types = [x.rsplit(".", 1)[1] for x in types if x.startswith("openmdao.")]
tset = set(types)
noentrypts = cset - tset
if noentrypts:
self.fail("the following Components are not registered using entry points: %s" % noentrypts)
def test_PythonSourceTreeAnalyser(self):
skipdirs = set(['test', 'docs', 'examples', 'optproblems',
'build', 'dist'])
psta = PythonSourceTreeAnalyser(self.startdirs,
direxclude=lambda d: d in skipdirs)
self.assertTrue('openmdao.main.component.Component' in
psta.graph['openmdao.main.container.Container'])
self.assertTrue('openmdao.main.assembly.Assembly' in
psta.graph['openmdao.main.component.Component'])
self.assertTrue('openmdao.main.datatypes.float.Float' in
psta.graph['openmdao.main.variable.Variable'])
comps = psta.find_inheritors('openmdao.main.component.Component')
icomps = psta.find_inheritors('IComponent')
self.assertTrue('openmdao.main.assembly.Assembly' in icomps)
comps.extend(psta.find_inheritors('openmdao.main.variable.Variable'))
comps.extend(psta.find_inheritors('traits.api.Array'))
comps = [x.rsplit('.',1)[1] for x in comps]
cset = set(comps)
excludes = set([
'Driver',
'DriverUsesDerivatives',
'DistributionCaseDriver',
'CaseIterDriverBase',
'PassthroughTrait',
'PassthroughProperty',
'OptProblem',
'TraitArray',
'Broadcast', # utility class for bliss2000
'SubSystemOpt', # utility class for bliss2000
'SubSystemObj' # utility class for bliss2000
])
cset = cset - excludes
from openmdao.main.api import get_available_types
types = set([x[0] for x in get_available_types()])
types = [x.rsplit('.',1)[1] for x in types if x.startswith('openmdao.')]
tset = set(types)
noentrypts = cset-tset
if noentrypts:
self.fail("the following Components are not registered using entry points: %s" % noentrypts)
def _find_all_plugins(searchdir):
"""Return a dict containing lists of each plugin type found, keyed by
plugin group name, e.g., openmdao.component, openmdao.variable, etc.
"""
dct = {}
psta = PythonSourceTreeAnalyser(searchdir)
comps = psta.find_inheritors('openmdao.main.component.Component')
comps.extend(psta.find_inheritors('openmdao.main.api.Component'))
comps = set(comps)
drivers = psta.find_inheritors('openmdao.main.driver.Driver')
drivers.extend(psta.find_inheritors('openmdao.main.api.Driver'))
drivers = set(drivers)
comps = comps - drivers
dct['openmdao.component'] = comps
dct['openmdao.driver'] = drivers
variables = psta.find_inheritors('openmdao.main.api.Variable')
variables.extend(psta.find_inheritors('openmdao.main.variable.Variable'))
dct['openmdao.variable'] = set(variables)
return dct
def test_PythonSourceTreeAnalyser(self):
try:
import openmdao.main
import openmdao.lib
except ImportError:
# don't perform this test if openmdao.main
# and openmdao.lib aren't present
raise SkipTest("this test requires openmdao.main and openmdao.lib")
startdirs = [os.path.dirname(openmdao.main.__file__),
os.path.dirname(openmdao.lib.__file__)]
psta = PythonSourceTreeAnalyser(startdirs, os.path.join('*','test','*'))
self.assertTrue('openmdao.main.component.Component' in
psta.graph['openmdao.main.container.Container'])
self.assertTrue('openmdao.main.assembly.Assembly' in
psta.graph['openmdao.main.component.Component'])
self.assertTrue('openmdao.lib.datatypes.float.Float' in
psta.graph['openmdao.main.variable.Variable'])
comps = psta.find_inheritors('openmdao.main.component.Component')
comps.extend(psta.find_inheritors('openmdao.main.variable.Variable'))
comps.extend(psta.find_inheritors('enthought.traits.api.Array'))
comps = [x.rsplit('.',1)[1] for x in comps]
comps.remove('Driver')
comps.remove('DriverUsesDerivatives')
comps.remove('CaseIterDriverBase')
comps.remove('PassthroughTrait')
comps.remove('PassthroughProperty')
from openmdao.main.api import get_available_types
groups = [ 'openmdao.component',
'openmdao.driver',
'openmdao.variable']
types = set([x[0] for x in get_available_types(groups)])
types = [x.rsplit('.',1)[1] for x in types if x.startswith('openmdao.')]
cset = set(comps)
tset = set(types)
noentrypts = cset-tset
if noentrypts:
self.fail("the following Components are not registered using entry points: %s" % noentrypts)
def __init__(self, watchdir, use_observer=True, observer=None):
super(ProjDirFactory, self).__init__()
self._lock = threading.RLock()
self.watchdir = watchdir
self.imported = {} # imported files vs (module, ctor dict)
try:
self.analyzer = PythonSourceTreeAnalyser()
added_set = set()
changed_set = set()
deleted_set = set()
for pyfile in find_files(self.watchdir, "*.py"):
self.on_modified(pyfile, added_set, changed_set, deleted_set)
if use_observer:
self._start_observer(observer)
self.publish_updates(added_set, changed_set, deleted_set)
else:
self.observer = None # sometimes for debugging/testing it's easier to turn observer off
except Exception as err:
logger.error(str(err))
def _find_all_plugins(searchdir):
"""Return a dict containing lists of each plugin type found, keyed by
plugin group name, e.g., openmdao.component, openmdao.variable, etc.
"""
dct = {}
modnames = [
'openmdao.main', 'openmdao.lib.datatypes', 'openmdao.lib.components',
'openmdao.lib.drivers', 'openmdao.lib.surrogatemodels',
'openmdao.lib.doegenerators', 'openmdao.lib.differentiators',
'openmdao.lib.optproblems', 'openmdao.lib.casehandlers',
'openmdao.lib.architectures'
]
modules = []
for mod in modnames:
try:
__import__(mod)
except ImportError:
print 'skipping import of %s' % mod
else:
modules.append(sys.modules[mod])
dirs = [os.path.dirname(m.__file__) for m in modules] + [searchdir]
psta = PythonSourceTreeAnalyser(dirs, exclude=_exclude_funct)
comps = psta.find_inheritors('IComponent')
comps = set(comps)
drivers = psta.find_inheritors('IDriver')
drivers = set(drivers)
comps = comps - drivers
dct['openmdao.component'] = comps
dct['openmdao.driver'] = drivers
variables = psta.find_inheritors('IVariable')
dct['openmdao.variable'] = set(variables)
return dct
def test_optproblems_solution(self):
#find all the optproblems in lib
startdirs = [os.path.dirname(openmdao.lib.optproblems.__file__),]
psta = PythonSourceTreeAnalyser(startdirs, os.path.join('*','test','*'))
opt_problems = psta.find_inheritors("openmdao.main.problem_formulation.OptProblem")
for prob_name in opt_problems:
#print "running %s"%prob_name
prob_class = prob_name.split(".")[-1]
prob_package = ".".join(prob_name.split(".")[:-1])
prob_package = __import__(prob_package,globals(),locals(),[prob_class,],-1)
prob = getattr(prob_package,prob_class)() #create instance of the OptProblem
prob.architecture = OptProblemSolutionCheck()
prob.configure()
prob.run()
error = prob.check_solution(strict=True)
self.assertAccuracy(prob_name,error,.001)
def test_PythonSourceTreeAnalyser(self):
skipdirs = set(
['test', 'docs', 'examples', 'optproblems', 'build', 'dist'])
psta = PythonSourceTreeAnalyser(self.startdirs,
direxclude=lambda d: d in skipdirs)
self.assertTrue('openmdao.main.component.Component' in
psta.graph['openmdao.main.container.Container'])
self.assertTrue('openmdao.main.assembly.Assembly' in
psta.graph['openmdao.main.component.Component'])
self.assertTrue('openmdao.main.datatypes.float.Float' in
psta.graph['openmdao.main.variable.Variable'])
comps = psta.find_inheritors('openmdao.main.component.Component')
icomps = psta.find_inheritors('IComponent')
self.assertTrue('openmdao.main.assembly.Assembly' in icomps)
comps.extend(psta.find_inheritors('openmdao.main.variable.Variable'))
comps.extend(psta.find_inheritors('traits.api.Array'))
comps = [x.rsplit('.', 1)[1] for x in comps]
cset = set(comps)
excludes = set([
'Driver',
'DriverUsesDerivatives',
'DistributionCaseDriver',
'CaseIterDriverBase',
'PassthroughTrait',
'PassthroughProperty',
'OptProblem',
'TraitArray',
'Broadcast', # utility class for bliss2000
'SubSystemOpt', # utility class for bliss2000
'SubSystemObj' # utility class for bliss2000
])
cset = cset - excludes
from openmdao.main.api import get_available_types
types = set([x[0] for x in get_available_types()])
types = [
x.rsplit('.', 1)[1] for x in types if x.startswith('openmdao.')
]
tset = set(types)
noentrypts = cset - tset
if noentrypts:
self.fail(
"the following Components are not registered using entry points: %s"
% noentrypts)
class ProjDirFactory(Factory):
"""A Factory that watches a Project directory and dynamically keeps
the set of available types up-to-date as project files change.
"""
def __init__(self, watchdir, use_observer=True, observer=None):
super(ProjDirFactory, self).__init__()
self._lock = threading.RLock()
self.watchdir = watchdir
self.imported = {} # imported files vs (module, ctor dict)
try:
self.analyzer = PythonSourceTreeAnalyser()
added_set = set()
changed_set = set()
deleted_set = set()
for pyfile in find_files(self.watchdir, "*.py"):
self.on_modified(pyfile, added_set, changed_set, deleted_set)
if use_observer:
self._start_observer(observer)
self.publish_updates(added_set, changed_set, deleted_set)
else:
self.observer = None # sometimes for debugging/testing it's easier to turn observer off
except Exception as err:
logger.error(str(err))
def _start_observer(self, observer):
if observer is None:
self.observer = Observer()
self._ownsobserver = True
else:
self.observer = observer
self._ownsobserver = False
self.observer.schedule(PyWatcher(self),
path=self.watchdir,
recursive=True)
if self._ownsobserver:
self.observer.daemon = True
self.observer.start()
def _get_mod_ctors(self, mod, fpath, visitor):
self.imported[fpath] = (mod, {})
for cname in visitor.classes.keys():
self.imported[fpath][1][cname] = getattr(mod, cname.split('.')[-1])
def create(self,
typ,
version=None,
server=None,
res_desc=None,
**ctor_args):
"""Create and return an instance of the specified type, or None if
this Factory can't satisfy the request.
"""
if server is None and res_desc is None and typ in self.analyzer.class_map:
with self._lock:
fpath = self.analyzer.class_map[typ].fname
modpath = self.analyzer.fileinfo[fpath][0].modpath
if os.path.getmtime(fpath) > self.analyzer.fileinfo[fpath][
1] and modpath in sys.modules:
reload(sys.modules[modpath])
if fpath not in self.imported:
sys.path = [
get_ancestor_dir(fpath, len(modpath.split('.')))
] + sys.path
try:
__import__(modpath)
except ImportError as err:
return None
finally:
sys.path = sys.path[1:]
mod = sys.modules[modpath]
visitor = self.analyzer.fileinfo[fpath][0]
self._get_mod_ctors(mod, fpath, visitor)
try:
ctor = self.imported[fpath][1][typ]
except KeyError:
return None
return ctor(**ctor_args)
return None
def get_available_types(self, groups=None):
"""Return a list of available types that cause predicate(classname, metadata) to
return True.
"""
with self._lock:
graph = self.analyzer.graph
typset = set(graph.nodes())
types = []
if groups is None:
ifaces = set([v[0] for v in plugin_groups.values()])
else:
ifaces = set(
[v[0] for k, v in plugin_groups.items() if k in groups])
for typ in typset:
if typ.startswith(
'openmdao.'): # don't include any standard lib types
continue
if 'classinfo' in graph.node[typ]:
meta = graph.node[typ]['classinfo'].meta
if ifaces.intersection(self.analyzer.get_interfaces(typ)):
meta = meta.copy()
meta['_context'] = 'In Project'
types.append((typ, meta))
return types
def on_modified(self, fpath, added_set, changed_set, deleted_set):
if os.path.isdir(fpath):
return
with self._lock:
imported = False
if fpath in self.analyzer.fileinfo: # file has been previously scanned
visitor = self.analyzer.fileinfo[fpath][0]
pre_set = set(visitor.classes.keys())
if fpath in self.imported: # we imported it earlier
imported = True
sys.path = [os.path.dirname(fpath)
] + sys.path # add fpath location to sys.path
try:
reload(self.imported[fpath][0])
except ImportError as err:
return None
finally:
sys.path = sys.path[1:] # restore original sys.path
#self.imported[fpath] = (m, self.imported[fpath][1])
elif os.path.getmtime(
fpath) > self.analyzer.fileinfo[fpath][1]:
modpath = get_module_path(fpath)
if modpath in sys.modules:
reload(sys.modules[modpath])
self.on_deleted(fpath, set()) # clean up old refs
else: # it's a new file
pre_set = set()
visitor = self.analyzer.analyze_file(fpath)
post_set = set(visitor.classes.keys())
deleted_set.update(pre_set - post_set)
added_set.update(post_set - pre_set)
if imported:
changed_set.update(pre_set.intersection(post_set))
def on_deleted(self, fpath, deleted_set):
with self._lock:
if os.path.isdir(fpath):
for pyfile in find_files(self.watchdir, "*.py"):
self.on_deleted(pyfile, deleted_set)
else:
try:
del self.imported[fpath]
except KeyError:
pass
visitor = self.analyzer.fileinfo[fpath][0]
deleted_set.update(visitor.classes.keys())
self.analyzer.remove_file(fpath)
def publish_updates(self, added_set, changed_set, deleted_set):
publisher = Publisher.get_instance()
if publisher:
types = get_available_types()
types.extend(self.get_available_types())
publisher.publish('types', [
packagedict(types),
list(added_set),
list(changed_set),
list(deleted_set),
])
else:
logger.error("no Publisher found")
def cleanup(self):
"""If this factory is removed from the FactoryManager during execution, this function
will stop the watchdog observer thread.
"""
if self.observer and self._ownsobserver:
self.observer.unschedule_all()
self.observer.stop()
self.observer.join()
class PkgResourcesFactory(Factory):
"""A Factory that loads plugins using the pkg_resources API, which means
it searches through egg info of distributions in order to find any entry
point groups corresponding to openmdao plugin types, e.g.,
openmdao.component, openmdao.variable, etc.
"""
def __init__(self, groups=plugin_groups.keys(), search_path=None):
super(PkgResourcesFactory, self).__init__()
self._have_new_types = True
self._groups = copy.copy(groups)
self._search_path = search_path
self.env = Environment(search_path)
self.tree_analyser = PythonSourceTreeAnalyser()
def create(self, typ, version=None, server=None,
res_desc=None, **ctor_args):
"""Create and return an object of the given type, with
optional name, version, server id, and resource description.
"""
if server is not None or res_desc is not None:
return None
klass = self._load(typ, version)
if klass is None:
return None
else:
return klass(**ctor_args)
def _load(self, typ, version):
"""Return class for *typ* and *version*."""
classes = self._get_type_dict()
try:
lst = classes[typ]
dist = lst[0]
groups = lst[1]
klass = dist.load_entry_point(groups[0], typ)
if version is not None and dist.version != version:
return None
return klass
except KeyError:
if self._search_path is None:
return None
# try to look in the whole environment
for group in self._groups:
for proj in self.env:
for dist in self.env[proj]:
if version is not None and version != dist.version:
continue
ep = dist.get_entry_info(group, typ)
if ep is not None:
dist.activate()
klass = ep.load(require=True, env=self.env)
self._have_new_types = True
return klass
if version is None:
# newest version didn't have entry point, so skip to next project
break
return None
def _entry_map_info(self, distiter):
dct = {}
for group in plugin_groups.keys():
for dist in distiter:
for name, value in dist.get_entry_map(group).items():
lst = dct.setdefault(name, (dist, [], set()))
lst[1].append(group)
lst[2].add(value.module_name)
return dct
def _get_type_dict(self):
if self._have_new_types:
self._entry_pt_classes = self._entry_map_info(working_set)
return self._entry_pt_classes
def _get_meta_info(self, typ_list, groups, typ_dict):
distset = set()
for name, lst in typ_dict.items():
dist = lst[0]
modules = lst[2]
distset.add(dist.project_name)
ifaces = set()
for g in lst[1]:
ifaces.update(plugin_groups[g])
meta = {
'version': dist.version,
'ifaces': set(ifaces),
}
for modname in modules:
fpath = find_module(modname)
if fpath is not None:
fanalyzer = self.tree_analyser.analyze_file(fpath, use_cache=True)
meta['bases'] = fanalyzer.classes[name].bases
meta['ifaces'].update(fanalyzer.classes[name].meta['ifaces'])
meta['ifaces'] = list(meta['ifaces'])
if groups.intersection(lst[1]):
typ_list.append((name, meta))
self.tree_analyser.flush_cache()
return distset
def get_available_types(self, groups=None):
"""Return a set of tuples of the form (typename, dist_version), one
for each available plugin type in the given entry point groups.
If groups is None, return the set for all openmdao entry point groups.
"""
ret = []
if groups is None:
groups = plugin_groups.keys()
groups = set(groups)
typ_dict = self._get_type_dict()
distset = self._get_meta_info(ret, groups, typ_dict)
if self._search_path is None: # self.env has same contents as working_set,
# so don't bother looking through it
return ret
# now look in the whole Environment
dists = [] # we want an iterator of newest dist for each project in Environment
for proj in self.env:
dist = self.env[proj][0]
if dist.project_name not in distset:
dists.append(dist)
typ_dict = self._entry_map_info(dists)
#dset = self._get_meta_info(ret, groups, typ_dict)
return ret
def get_signature(self, typname, version=None):
"""Return constructor argument signature for *typname,* using the
specified package version. The return value is a dictionary.
"""
cls = self._load(typname, version)
if cls is None:
return None
else:
return self.form_signature(cls)
def test_PythonSourceTreeAnalyser(self):
try:
import openmdao.main
import openmdao.lib
except ImportError:
# don't perform this test if openmdao.main
# and openmdao.lib aren't present
raise SkipTest("this test requires openmdao.main and openmdao.lib")
def exclude_tests(pname):
parts = pname.split(os.sep)
return 'test' in parts
startdirs = [
os.path.dirname(openmdao.main.__file__),
os.path.dirname(openmdao.lib.__file__)
]
psta = PythonSourceTreeAnalyser(startdirs, exclude_tests)
self.assertTrue('openmdao.main.component.Component' in
psta.graph['openmdao.main.container.Container'])
self.assertTrue('openmdao.main.assembly.Assembly' in
psta.graph['openmdao.main.component.Component'])
self.assertTrue('openmdao.main.datatypes.float.Float' in
psta.graph['openmdao.main.variable.Variable'])
comps = psta.find_inheritors('openmdao.main.component.Component')
icomps = psta.find_inheritors('IComponent')
self.assertTrue('openmdao.main.assembly.Assembly' in icomps)
comps.extend(psta.find_inheritors('openmdao.main.variable.Variable'))
comps.extend(psta.find_inheritors('enthought.traits.api.Array'))
comps = [
x.rsplit('.', 1)[1] for x in comps
if '.examples.' not in x and '.optproblems.' not in x
]
cset = set(comps)
excludes = set([
'Driver',
'DriverUsesDerivatives',
'DistributionCaseDriver',
'CaseIterDriverBase',
'PassthroughTrait',
'PassthroughProperty',
'OptProblem',
'TraitArray',
])
cset = cset - excludes
from openmdao.main.api import get_available_types
types = set([x[0] for x in get_available_types()])
types = [
x.rsplit('.', 1)[1] for x in types if x.startswith('openmdao.')
]
tset = set(types)
noentrypts = cset - tset
if noentrypts:
self.fail(
"the following Components are not registered using entry points: %s"
% noentrypts)
