class ProjDirFactory(Factory): """A Factory that watches a Project directory and dynamically keeps the set of available types up-to-date as project files change. """ def __init__(self, watchdir, use_observer=True, observer=None): super(ProjDirFactory, self).__init__() self._lock = threading.RLock() self.watchdir = watchdir self.imported = {} # imported files vs (module, ctor dict) try: self.analyzer = PythonSourceTreeAnalyser() added_set = set() changed_set = set() deleted_set = set() for pyfile in find_files(self.watchdir, "*.py"): self.on_modified(pyfile, added_set, changed_set, deleted_set) if use_observer: self._start_observer(observer) self.publish_updates(added_set, changed_set, deleted_set) else: self.observer = None # sometimes for debugging/testing it's easier to turn observer off except Exception as err: logger.error(str(err)) def _start_observer(self, observer): if observer is None: self.observer = Observer() self._ownsobserver = True else: self.observer = observer self._ownsobserver = False self.observer.schedule(PyWatcher(self), path=self.watchdir, recursive=True) if self._ownsobserver: self.observer.daemon = True self.observer.start() def _get_mod_ctors(self, mod, fpath, visitor): self.imported[fpath] = (mod, {}) for cname in visitor.classes.keys(): self.imported[fpath][1][cname] = getattr(mod, cname.split('.')[-1]) def create(self, typ, version=None, server=None, res_desc=None, **ctor_args): """Create and return an instance of the specified type, or None if this Factory can't satisfy the request. """ if server is None and res_desc is None and typ in self.analyzer.class_map: with self._lock: fpath = self.analyzer.class_map[typ].fname modpath = self.analyzer.fileinfo[fpath][0].modpath if os.path.getmtime(fpath) > self.analyzer.fileinfo[fpath][ 1] and modpath in sys.modules: reload(sys.modules[modpath]) if fpath not in self.imported: sys.path = [ get_ancestor_dir(fpath, len(modpath.split('.'))) ] + sys.path try: __import__(modpath) except ImportError as err: return None finally: sys.path = sys.path[1:] mod = sys.modules[modpath] visitor = self.analyzer.fileinfo[fpath][0] self._get_mod_ctors(mod, fpath, visitor) try: ctor = self.imported[fpath][1][typ] except KeyError: return None return ctor(**ctor_args) return None def get_available_types(self, groups=None): """Return a list of available types that cause predicate(classname, metadata) to return True. """ with self._lock: graph = self.analyzer.graph typset = set(graph.nodes()) types = [] if groups is None: ifaces = set([v[0] for v in plugin_groups.values()]) else: ifaces = set( [v[0] for k, v in plugin_groups.items() if k in groups]) for typ in typset: if typ.startswith( 'openmdao.'): # don't include any standard lib types continue if 'classinfo' in graph.node[typ]: meta = graph.node[typ]['classinfo'].meta if ifaces.intersection(self.analyzer.get_interfaces(typ)): meta = meta.copy() meta['_context'] = 'In Project' types.append((typ, meta)) return types def on_modified(self, fpath, added_set, changed_set, deleted_set): if os.path.isdir(fpath): return with self._lock: imported = False if fpath in self.analyzer.fileinfo: # file has been previously scanned visitor = self.analyzer.fileinfo[fpath][0] pre_set = set(visitor.classes.keys()) if fpath in self.imported: # we imported it earlier imported = True sys.path = [os.path.dirname(fpath) ] + sys.path # add fpath location to sys.path try: reload(self.imported[fpath][0]) except ImportError as err: return None finally: sys.path = sys.path[1:] # restore original sys.path #self.imported[fpath] = (m, self.imported[fpath][1]) elif os.path.getmtime( fpath) > self.analyzer.fileinfo[fpath][1]: modpath = get_module_path(fpath) if modpath in sys.modules: reload(sys.modules[modpath]) self.on_deleted(fpath, set()) # clean up old refs else: # it's a new file pre_set = set() visitor = self.analyzer.analyze_file(fpath) post_set = set(visitor.classes.keys()) deleted_set.update(pre_set - post_set) added_set.update(post_set - pre_set) if imported: changed_set.update(pre_set.intersection(post_set)) def on_deleted(self, fpath, deleted_set): with self._lock: if os.path.isdir(fpath): for pyfile in find_files(self.watchdir, "*.py"): self.on_deleted(pyfile, deleted_set) else: try: del self.imported[fpath] except KeyError: pass visitor = self.analyzer.fileinfo[fpath][0] deleted_set.update(visitor.classes.keys()) self.analyzer.remove_file(fpath) def publish_updates(self, added_set, changed_set, deleted_set): publisher = Publisher.get_instance() if publisher: types = get_available_types() types.extend(self.get_available_types()) publisher.publish('types', [ packagedict(types), list(added_set), list(changed_set), list(deleted_set), ]) else: logger.error("no Publisher found") def cleanup(self): """If this factory is removed from the FactoryManager during execution, this function will stop the watchdog observer thread. """ if self.observer and self._ownsobserver: self.observer.unschedule_all() self.observer.stop() self.observer.join()
class PkgResourcesFactory(Factory): """A Factory that loads plugins using the pkg_resources API, which means it searches through egg info of distributions in order to find any entry point groups corresponding to openmdao plugin types, e.g., openmdao.component, openmdao.variable, etc. """ def __init__(self, groups=plugin_groups.keys(), search_path=None): super(PkgResourcesFactory, self).__init__() self._have_new_types = True self._groups = copy.copy(groups) self._search_path = search_path self.env = Environment(search_path) self.tree_analyser = PythonSourceTreeAnalyser() def create(self, typ, version=None, server=None, res_desc=None, **ctor_args): """Create and return an object of the given type, with optional name, version, server id, and resource description. """ if server is not None or res_desc is not None: return None klass = self._load(typ, version) if klass is None: return None else: return klass(**ctor_args) def _load(self, typ, version): """Return class for *typ* and *version*.""" classes = self._get_type_dict() try: lst = classes[typ] dist = lst[0] groups = lst[1] klass = dist.load_entry_point(groups[0], typ) if version is not None and dist.version != version: return None return klass except KeyError: if self._search_path is None: return None # try to look in the whole environment for group in self._groups: for proj in self.env: for dist in self.env[proj]: if version is not None and version != dist.version: continue ep = dist.get_entry_info(group, typ) if ep is not None: dist.activate() klass = ep.load(require=True, env=self.env) self._have_new_types = True return klass if version is None: # newest version didn't have entry point, so skip to next project break return None def _entry_map_info(self, distiter): dct = {} for group in plugin_groups.keys(): for dist in distiter: for name, value in dist.get_entry_map(group).items(): lst = dct.setdefault(name, (dist, [], set())) lst[1].append(group) lst[2].add(value.module_name) return dct def _get_type_dict(self): if self._have_new_types: self._entry_pt_classes = self._entry_map_info(working_set) return self._entry_pt_classes def _get_meta_info(self, typ_list, groups, typ_dict): distset = set() for name, lst in typ_dict.items(): dist = lst[0] modules = lst[2] distset.add(dist.project_name) ifaces = set() for g in lst[1]: ifaces.update(plugin_groups[g]) meta = { 'version': dist.version, 'ifaces': set(ifaces), } for modname in modules: fpath = find_module(modname) if fpath is not None: fanalyzer = self.tree_analyser.analyze_file(fpath, use_cache=True) meta['bases'] = fanalyzer.classes[name].bases meta['ifaces'].update(fanalyzer.classes[name].meta['ifaces']) meta['ifaces'] = list(meta['ifaces']) if groups.intersection(lst[1]): typ_list.append((name, meta)) self.tree_analyser.flush_cache() return distset def get_available_types(self, groups=None): """Return a set of tuples of the form (typename, dist_version), one for each available plugin type in the given entry point groups. If groups is None, return the set for all openmdao entry point groups. """ ret = [] if groups is None: groups = plugin_groups.keys() groups = set(groups) typ_dict = self._get_type_dict() distset = self._get_meta_info(ret, groups, typ_dict) if self._search_path is None: # self.env has same contents as working_set, # so don't bother looking through it return ret # now look in the whole Environment dists = [] # we want an iterator of newest dist for each project in Environment for proj in self.env: dist = self.env[proj][0] if dist.project_name not in distset: dists.append(dist) typ_dict = self._entry_map_info(dists) #dset = self._get_meta_info(ret, groups, typ_dict) return ret def get_signature(self, typname, version=None): """Return constructor argument signature for *typname,* using the specified package version. The return value is a dictionary. """ cls = self._load(typname, version) if cls is None: return None else: return self.form_signature(cls)