def add_javascript_files(self, fragment, item): """ Add all the JavaScript files from a directory to the specified fragment """ if pkg_resources.resource_isdir(__name__, item): for child_item in pkg_resources.resource_listdir(__name__, item): path = os.path.join(item, child_item) if not pkg_resources.resource_isdir(__name__, path): fragment.add_javascript_url(self.runtime.local_resource_url(self, path)) else: fragment.add_javascript_url(self.runtime.local_resource_url(self, item))
def load_collection(self, package, path, keyname=None, display_name=None): if not keyname: keyname = path.split("/")[-1] if not display_name: display_name = keyname try: collection = MarkerCollection.filter_by(keyname=keyname).one() except NoResultFound: collection = MarkerCollection(keyname=keyname, display_name=display_name).persist() for catname in resource_listdir(package, path): if not resource_isdir(package, path + "/" + catname): continue try: category = MarkerCategory.filter_by(keyname=catname).one() except NoResultFound: category = MarkerCategory(keyname=catname, display_name=catname).persist() for fn in resource_listdir(package, path + "/" + catname): if not fn.endswith(".svg"): continue if resource_isdir(package, path + "/" + catname + "/" + fn): continue mkeyname = re.sub(r"\.svg$", "", fn) try: marker = Marker.filter_by(keyname=mkeyname).one() assert marker.collection == collection, "Marker '%s' found in collection '%s'!" % ( mkeyname, marker.collection.keyname, ) assert marker.category == category, "Marker '%s' found in category '%s'!" % ( mkeyname, marker.category.keyname, ) except NoResultFound: marker = Marker( collection=collection, category=category, keyname=mkeyname, display_name=mkeyname ).persist() marker.load_file(resource_stream(package, path + "/" + catname + "/" + fn))
def __call__(self, context, request): if self.use_subpath: path_tuple = request.subpath else: path_tuple = traversal_path_info(request.environ['PATH_INFO']) path = _secure_path(path_tuple) if path is None: raise HTTPNotFound('Out of bounds: %s' % request.url) if self.package_name: # package resource resource_path ='%s/%s' % (self.docroot.rstrip('/'), path) if resource_isdir(self.package_name, resource_path): if not request.path_url.endswith('/'): self.add_slash_redirect(request) resource_path = '%s/%s' % (resource_path.rstrip('/'),self.index) if not resource_exists(self.package_name, resource_path): raise HTTPNotFound(request.url) filepath = resource_filename(self.package_name, resource_path) else: # filesystem file # os.path.normpath converts / to \ on windows filepath = normcase(normpath(join(self.norm_docroot, path))) if isdir(filepath): if not request.path_url.endswith('/'): self.add_slash_redirect(request) filepath = join(filepath, self.index) if not exists(filepath): raise HTTPNotFound(request.url) return FileResponse(filepath, request, self.cache_max_age)
def __place_template_folder(group, src, dst, gbp=False): template_files = pkg_resources.resource_listdir(group, src) # For each template, place for template_file in template_files: template_path = os.path.join(src, template_file) template_dst = os.path.join(dst, template_file) if pkg_resources.resource_isdir(group, template_path): debug("Recursing on folder '{0}'".format(template_path)) __place_template_folder(group, template_path, template_dst, gbp) else: try: debug("Placing template '{0}'".format(template_path)) template = pkg_resources.resource_string(group, template_path) template_abs_path = pkg_resources.resource_filename(group, template_path) except IOError as err: error("Failed to load template " "'{0}': {1}".format(template_file, str(err)), exit=True) if not os.path.exists(dst): os.makedirs(dst) if os.path.exists(template_dst): debug("Removing existing file '{0}'".format(template_dst)) os.remove(template_dst) with open(template_dst, 'w') as f: if not isinstance(template, str): template = template.decode('utf-8') f.write(template) shutil.copystat(template_abs_path, template_dst)
def isfile(name, source=None): """ Return True if the path is a file, False otherwise. """ if os.path.isabs(name): return os.path.isfile(name) # Iterate to find the path. if isinstance(source, (tuple, list)): sources = list(source) else: sources = [source] if source else _sources for src in sources: if isinstance(src, basestring) and not src.startswith("py:"): if os.path.isfile(os.path.join(src, name)): return True continue # Must be a ``pkg_resources`` thing. assert_pkg_resources() if isinstance(src, basestring): src = src[3:] # Is it a file? if pkg_resources.resource_exists(src, name) and not pkg_resources.resource_isdir(src, name): return True return False
def iter_content_resource(self, root='content/toolkit'): for item in resource_listdir('org.bccvl.compute', root): item = os.path.join(root, item) yield item if resource_isdir('org.bccvl.compute', item): for subitem in self.iter_content_resource(item): yield subitem
def extract_dirs(mod, path, dst, verbose=False, exclude=None, exclude_ext=None, recursion=True, replace=True): """ mod name path mod path dst output directory resursion True will extract all sub module of mod """ default_exclude = [".svn", "_svn", ".git"] default_exclude_ext = [".pyc", ".pyo", ".bak", ".tmp"] exclude = exclude or [] exclude_ext = exclude_ext or [] # log = logging.getLogger('uliweb') if not os.path.exists(dst): os.makedirs(dst) if verbose: print "Make directory %s" % dst for r in pkg.resource_listdir(mod, path): if r in exclude or r in default_exclude: continue fpath = os.path.join(path, r) if pkg.resource_isdir(mod, fpath): if recursion: extract_dirs(mod, fpath, os.path.join(dst, r), verbose, exclude, exclude_ext, recursion, replace) else: ext = os.path.splitext(fpath)[1] if ext in exclude_ext or ext in default_exclude_ext: continue extract_file(mod, fpath, dst, verbose, replace)
def extract_folder(mod, path, dst, exclude=None, exclude_ext=None, recursion=True, replace=True, callback=None): default_exclude = ['.svn', '_svn', '.git', '.hg'] default_exclude_ext = ['.pyc', '.pyo', '.bak', '.tmp'] exclude = exclude or [] exclude_ext = exclude_ext or [] if callback is None: callback = _callback if not os.path.exists(dst): os.makedirs(dst) if callback and callable(callback): callback(dst, action='extract_folder') for r in pkg.resource_listdir(mod, path): if r in exclude or r in default_exclude: continue fpath = os.path.join(path, r) if pkg.resource_isdir(mod, fpath): if recursion: extract_folder(mod, fpath, os.path.join(dst, r), exclude, exclude_ext, recursion, replace, callback) else: ext = os.path.splitext(fpath)[1] if ext in exclude_ext or ext in default_exclude_ext: continue extract_file(mod, fpath, dst, replace, callback)
def _walk_dir(aliases, *parts): root_name = build_index_name(*parts) resource_name = os.path.join(*parts) if root_name not in aliases: self.number_of_indexes += 1 data = aliases.get(root_name, {}) for filename in resource_listdir(package_name, resource_name): index_name = build_index_name(*(parts + (filename, ))) file_path = os.path.join(resource_name, filename) if resource_isdir(package_name, file_path): _walk_dir(data, *(parts + (filename, ))) continue ext = os.path.splitext(filename)[1] if ext not in {'.json', }: continue assert index_name not in data, 'Duplicate index' data[index_name] = self.mappings[index_name] = \ resource_filename( package_name, os.path.join(resource_name, filename)) self.number_of_indexes += 1 aliases[root_name] = data
def add_entrypoint(self, entrypoint): """Load translations from an entrypoint.""" for ep in iter_entry_points(group=entrypoint): if not resource_isdir(ep.module_name, 'translations'): continue dirname = resource_filename(ep.module_name, 'translations') self.add_path(dirname)
def _unpack_resource(self, resource_path, resource_name, resource_executable): if not pkg_resources.resource_exists(__name__, resource_name): return if pkg_resources.resource_isdir(__name__, resource_name): self.__create_dir(resource_path) for f in pkg_resources.resource_listdir(__name__, resource_name): if f == "": # TODO(beng): Figure out why this happens continue # TODO: Handle executable resources in directory self._unpack_resource( os.path.join(resource_path, f), os.path.join(resource_name, f), False, ) else: with closable_named_temporary_file( prefix=resource_path + os.extsep ) as outf: outf.write(pkg_resources.resource_string(__name__, resource_name)) if resource_executable and hasattr(os, "fchmod"): st = os.fstat(outf.fileno()) os.fchmod(outf.fileno(), st.st_mode | stat.S_IXUSR) outf.close() shutil.copy(outf.name, resource_path)
def _unpack_dir(self, resource_dir, dst_dir): if not pkg_resources.resource_exists(__name__, resource_dir): raise Exception( "Cannot unpack directory: {0} doesn't exist in the package".format( resource_dir ) ) if not pkg_resources.resource_isdir(__name__, resource_dir): raise Exception( "Cannot unpack directory: {0} is not a directory".format(resource_dir) ) self.__create_dir(dst_dir) if not os.path.exists(dst_dir): raise Exception( "Cannot unpack directory: cannot create directory {0}".format(dst_dir) ) for resource_file in pkg_resources.resource_listdir(__name__, resource_dir): resource_path = os.path.join(dst_dir, resource_file) if os.path.exists(resource_path): continue self._unpack_resource( resource_path, "/".join((resource_dir, resource_file)), False )
def init_extensions(app): """ Load the Redash extensions for the given Redash Flask app. """ if not hasattr(app, 'redash_extensions'): app.redash_extensions = {} for entry_point in iter_entry_points('redash.extensions'): app.logger.info('Loading Redash extension %s.', entry_point.name) try: extension = entry_point.load() app.redash_extensions[entry_point.name] = { "entry_function": extension(app), "resources_list": [] } except ImportError: app.logger.info('%s does not have a callable and will not be loaded.', entry_point.name) (root_module, _) = os.path.splitext(entry_point.module_name) content_folder_relative = os.path.join(entry_point.name, 'bundle') # If it's a frontend extension only, store a list of files in the bundle directory. if resource_isdir(root_module, content_folder_relative): app.redash_extensions[entry_point.name] = { "entry_function": None, "resources_list": resource_listdir(root_module, content_folder_relative) }
def extract_dirs(mod, path, dst, verbose=False, exclude=None, exclude_ext=None, recursion=True, replace=True): default_exclude = ['.git'] default_exclude_ext = ['.pyc', '.pyo', '.bak', '.tmp', '.swp'] exclude = exclude or [] exclude_ext = exclude_ext or [] if not os.path.exists(dst): os.makedirs(dst) if verbose: print 'Make directory %s' %dst for r in pkg.resource_listdir(mod, path): if r in exclude or r in default_exclude: continue fpath =os.path.join(path, r) if pkg.resource_isdir(mod, fpath): if recursion: extract_dirs(mod, fpath, os.path.join(dst, r), verbose, exclude, exclude_ext, recursion, replace) else: ext = os.path.splitext(fpath)[1] if ext in exclude_ext or ext in default_exclude_ext: continue extract_file(mod, fpath, dst, verbose, replace)
def _resource_or_bust(self, path, isdir=False): """Returns the path to a resource, or None if it doesn't exist.""" if resource_exists(self.module_name, path) and \ resource_isdir(self.module_name, path) == isdir: return resource_filename(self.module_name, path) return None
def listdir(name, source=None): """ Generate the entries at a given path. """ if os.path.isabs(name): for entry in os.listdir(name): yield entry return # Figure out what we're iterating. if isinstance(source, (tuple, list)): sources = list(source) else: sources = [source] if source else _sources # Now, build it. for src in sources: if isinstance(src, basestring) and not src.startswith("py:"): path = os.path.join(src, name) if not os.path.isdir(path): continue this_src = os.listdir(path) else: if isinstance(src, basestring) and src.startswith("py:"): src = src[3:] if not pkg_resources.resource_isdir(src, name): continue this_src = pkg_resources.resource_listdir(src, name) for entry in this_src: yield entry
def generate_pkg_resources(pkg, module_name, re_filename=None): """Generate list of package resourses.""" pkg_dirname = os.path.dirname(importlib.import_module(pkg).__file__) if resource_isdir(pkg, module_name): for filename in resource_listdir(pkg, module_name): if re_filename is None or re_filename.match(filename): yield os.path.join(pkg_dirname, module_name, filename)
def templates(cls): """ Returns a list of Template objects that describe possible templates that can be used to create a module of this type. If no templates are provided, there will be no way to create a module of this type Expects a class attribute template_dir_name that defines the directory inside the 'templates' resource directory to pull templates from """ templates = [] dirname = os.path.join('templates', cls.template_dir_name) if not resource_isdir(__name__, dirname): log.warning("No resource directory {dir} found when loading {cls_name} templates".format( dir=dirname, cls_name=cls.__name__, )) return [] for template_file in resource_listdir(__name__, dirname): if not template_file.endswith('.yaml'): log.warning("Skipping unknown template file %s" % template_file) continue template_content = resource_string(__name__, os.path.join(dirname, template_file)) template = yaml.safe_load(template_content) templates.append(Template(**template)) return templates
def get_categories(category=None): if category is None: return resource_listdir(__name__, "data") else: return [item for item in resource_listdir(__name__, "data/" + category) if resource_isdir(__name__, "data/" + category + "/" + item)]
def _copy_resource_tree(self, modname, fname): try: for name in pkg_resources.resource_listdir(modname, fname): if name in self.IGNORED_NAMES: continue name = '/'.join((fname, name)) rel_name = '/'.join((modname, name)) if pkg_resources.resource_isdir(modname, name): self.execute(self._copy_resource_tree, (modname, name), "Recursing into " + rel_name) else: full_name = pkg_resources.resource_filename(modname, name) ct, _ = mimetypes.guess_type(full_name) require_once = None if self.requireonce and ct == "application/javascript": require_once = _JavascriptFileIter._marker_name(modname, name) stream = pkg_resources.resource_stream(modname, name) filename = '/'.join((modname, name)) self.execute(self.writer.write_file, (stream, filename), "Processing " + filename) if require_once is not None: filename = os.path.join(self.tempdir, filename) inf = open(filename) outname = tempfile.mktemp() outf = open(outname, "w") outf.write(_JavascriptFileIter.START_TEMPLATE % require_once) outf.write(inf.read()) outf.write(_JavascriptFileIter.END_TEMPLATE % require_once) outf.close() os.rename(outname, filename) stream.close() except OSError, e: if e.errno == errno.ENOENT: self.warn("Could not copy %s" % repr((modname, fname, e)))
def extract_dirs(mod, path, dst, verbose=False, exclude=None, exclude_ext=None, recursion=True): """ mod name path mod path dst output directory resursion True will extract all sub module of mod """ default_exclude = ['.svn', '_svn', '.git'] default_exclude_ext = ['.pyc', '.pyo', '.bak', '.tmp'] exclude = exclude or [] exclude_ext = exclude_ext or [] log = logging.getLogger('uliweb.console') if not os.path.exists(dst): os.makedirs(dst) if verbose: log.info('Make directory %s' % dst) for r in pkg.resource_listdir(mod, path): if r in exclude or r in default_exclude: continue fpath = os.path.join(path, r) if pkg.resource_isdir(mod, fpath): if recursion: extract_dirs(mod, fpath, os.path.join(dst, r), verbose, exclude, exclude_ext) else: ext = os.path.splitext(fpath)[1] if ext in exclude_ext or ext in default_exclude_ext: continue extract_file(mod, fpath, dst, verbose)
def get_file(path, mode='rb', source=SOURCE_ANY): """ Find and open the file at the given ``path``. This function first searches in the active profile's directory. If the file is not found there, and configuration is available, it attempts to use `pkg_resources <http://packages.python.org/distribute/pkg_resources.html>`_ to locate the file, otherwise it will search the root directory. If a file cannot be found, return ``None``. **Note:** ``mode`` is not used when getting a file from ``pkg_resources``. """ if os.path.isabs(path): if os.path.isfile(path): return open(path, mode) return None # Not an absolute path, so process it. ensure_paths() if source & SOURCE_PROFILE: p_path = os.path.join(profile_path, path) if os.path.isfile(p_path): return open(p_path, mode) if source & SOURCE_PKG_RESOURCES: if (package and pr and pr.resource_exists(package, path) and not pr.resource_isdir(package, path)): return pr.resource_stream(package, path) if source & SOURCE_ROOT: r_path = os.path.join(root_path, path) if os.path.isfile(r_path): return open(r_path, mode)
def static_content(environ, start_response): path = getpath(environ) if not path: # redirect start_response('301 Redirect', [content_type(), ('Location', environ['REQUEST_URI'] + '/')]) return [] if path == '/': path = 'web/index.html' # map / to index.html else: path = ('web/' + path).replace('//', '/') if path == 'web/js': # combined java scripts scripts = {} for s in resource_listdir('ctplot', 'web/js'): scripts[s] = '\n// {}\n\n'.format(s) + resource_string('ctplot', 'web/js/' + s) start_response('200 OK', [content_type('combined.js'), cc_cache]) return [scripts[k] for k in sorted(scripts.keys())] if not resource_exists('ctplot', path): # 404 start_response('404 Not Found', [content_type()]) return ['404\n', '{} not found!'.format(path)] elif resource_isdir('ctplot', path): # 403 start_response('403 Forbidden', [content_type()]) return ['403 Forbidden'] else: start_response('200 OK', [content_type(path), cc_cache]) return resource_string('ctplot', path)
def __init__(self, pkg_name, path, pattern='*.config', leaf="", base_path=None, prefix=None): #lg.setLevel(logging.DEBUG) lg.debug("pkg loading {} {} {}".format(pkg_name, path, pattern)) if not base_path is None: if leaf: leaf = leaf.strip('.') + '.' leaf = path.replace(base_path, '').strip('/').replace('/', '.') lg.debug("leaf: ({}) {}".format(base_path, leaf)) if not pkg_resources.resource_isdir(pkg_name, path): #asssume a file: lg.debug("loading file {} {}".format(pkg_name, path)) #print("loading file {} {}".format(pkg_name, path)) y = pkg_resources.resource_string(pkg_name, path) self[leaf].update(yaml.load(y)) else: for d in pkg_resources.resource_listdir(pkg_name, path): nres = os.path.join(path, d) #print('nn', nres, leaf) lg.debug("checking for pkg load: {}".format(nres)) if pkg_resources.resource_isdir(pkg_name, nres): lg.debug("pkg load: is directory: {}".format(nres)) if base_path == None: base_path = path #print('d', leaf, pkg_name, nres) y = YacoPkgDir(pkg_name, nres, pattern=pattern, base_path=base_path) self[leaf].update(y) else: if not fnmatch.fnmatch(d, pattern): lg.debug('ignoring {}'.format(nres)) continue else: lg.debug("pkg load: loading file: {}".format(nres)) y = yaml.load(pkg_resources.resource_string( pkg_name, nres)) lg.debug("pkg load: got: {}".format(str(y))) this_leaf = _get_leaf(leaf, d, pattern) #print('f', leaf, path, nres, d, this_leaf) self[this_leaf].update(y)
def get_templatesdir(self): """ Return the location of the templates director. Default implementation return the "templates" director if exists. Otherwise return None. """ if pkg_resources.resource_isdir(self.__module__, 'templates'): # @UndefinedVariable return pkg_resources.resource_filename(self.__module__, 'templates') # @UndefinedVariable return False
def __getitem__(self, name): subname = self.directory + name if resource_isdir(self.package, subname): return type(self)(self.package, subname) elif resource_exists(self.package, subname): file_ = resource_stream(self.package, subname) return Resource(file_) raise KeyError(name)
def get_staticdir(self): """ Return the location of the static directory. Default implementation return the "static" directory if exists. Otherwise return None. """ if pkg_resources.resource_isdir(self.__module__, 'static'): # @UndefinedVariable return pkg_resources.resource_filename(self.__module__, 'static') # @UndefinedVariable return False
def resources_for_test_config(test_config): resources = {} for key in [constants.CONFIG_BUNDLES_KEY, 'apps', 'libs', 'services']: key_path = 'test_configs/{}/{}'.format(test_config, key) if resource_isdir(__name__, key_path): resources[key] = {resource_name: resource_string(__name__, '{}/{}'.format(key_path, resource_name)) for resource_name in resource_listdir(__name__, key_path)} return resources
def create_project(shortname, dest, path='skel', **config): for filename in pkg.resource_listdir('modu', path): if(filename.startswith('__init__.py')): continue resource_path = os.path.join(path, filename) if(pkg.resource_isdir('modu', resource_path)): directory = filename if(directory.startswith('.')): continue if(directory.startswith('project')): directory = directory.replace('project', shortname) new_dest = os.path.join(dest, directory) if not(os.path.isdir(new_dest)): os.makedirs(new_dest) create_project(shortname, new_dest, resource_path, **config) else: if(filename.startswith('.')): continue new_file = None output_file = None file_stream = pkg.resource_stream('modu', resource_path) try: if(filename.endswith('.tmpl')): template_class = Template.compile(file=file_stream) variables = dict( project_name = shortname, project_description = config.get('longname', shortname), copyright_holder = config.get('author', shortname), year = time.strftime('%Y') ) output = str(template_class(searchList=[variables])) new_filename = filename[:-5] if(new_filename.startswith('project')): new_filename = new_filename.replace('project', shortname) new_path = os.path.join(dest, new_filename) new_file = open(new_path, 'w') new_file.write(output) new_file.close() else: # copy the file output_filename = os.path.join(dest, filename) output_file = open(output_filename, 'w') shutil.copyfileobj(file_stream, output_file) output_file.close() finally: if(new_file and not new_file.closed): new_file.close() if(output_file and not output_file.closed): output_file.close() file_stream.close()
def resource_names(d="."): for n in resource_listdir(__name__, d): r = d + "/" + n if resource_isdir(__name__, r): for n2 in resource_names(r): yield n2 elif is_html_resource(r): yield r
def get_templatesdir(self): """ Return the location of the templates director. Default implementation return the "templates" director if exists. Otherwise return None. """ if pkg_resources.resource_isdir(self.__module__, 'templates'): # @UndefinedVariable return pkg_resources.resource_filename( self.__module__, 'templates') # @UndefinedVariable return False
def _get_translation_directories(self, enabled_plugins): main_translation_directory = 'translations' result = [main_translation_directory] entry_points = (e for e in iter_entry_points(group='wazo_ui.plugins') if e.name in enabled_plugins) for ep in entry_points: if resource_isdir(ep.module_name, TRANSLATION_DIRECTORY): result.append( resource_filename(ep.module_name, TRANSLATION_DIRECTORY)) return result
def __getattr__(self, attr): file_loc = "data/" + self.directory + "/" + attr + ".json" dir_loc = "data/" + self.directory + "/" + attr if resource_exists(__name__, file_loc): return fetch_resource(file_loc) elif resource_exists(__name__, dir_loc) and \ resource_isdir(__name__, dir_loc): return CorpusLoader(self.directory + "/" + attr) else: raise AttributeError("no resource named " + attr)
def __init__(self, package, name, ignore=ignore): self._package = package self._name = name self.ignore = base_ignore + tuple(ignore) subdirs = [] for entry in self._listEntries(): entry_subpath = self._getEntrySubpath(entry) if resource_isdir(package, entry_subpath): subdirs.append(entry) self.subdirs = tuple(subdirs)
def copy_dir(d): os.makedirs(basedir + "/" + d, exist_ok=True) for each in pr.resource_listdir(package, d): filename = d + "/" + each if pr.resource_isdir(package, filename): copy_dir(filename) else: s = pr.resource_string(package, filename) with open(basedir + "/" + filename, "wb") as f: f.write(s)
def register_template_layer(config, resource_spec_dir, prefix=""): """Add a templates to the templates registry potentially overridin any previous tempaltes""" resource_spec_dir = config.absolute_asset_spec(resource_spec_dir) package, path = resource_spec_dir.split(":") for item in pkg_resources.resource_listdir(package, path): item_path = os.path.join(path, item) if not pkg_resources.resource_isdir(package, item_path): template_key = item.split(".")[0] template_spec = os.path.join(resource_spec_dir, item) config.registry["templates"][prefix + template_key] = template_spec
def get_package(package_name): try: pkg = pkg_resources.get_distribution(package_name) except pkg_resources.DistributionNotFound: raise click.UsageError('%s is not installed.' % (package_name, )) if not pkg_resources.resource_isdir(pkg.key, 'templates'): raise click.UsageError('%s does not have a templates directory.' % (pkg.key, )) return pkg
def resource_exists(cls, loadable): if pkg_resources is None: return False module_base, module_file = os.path.split(loadable) module_base = module_base.replace(os.sep, '.') try: return pkg_resources.resource_exists(module_base, module_file) and ( not pkg_resources.resource_isdir(module_base, module_file)) except (ValueError, ImportError): return False
def _install_scan_dir(package, src_dir, dst_dir, params, prefix_len, rec=None): """Interpolate source directory as a scan directory containing service definitions. """ package_name = package.__name__ src_control_dir = os.path.join(src_dir, _CONTROL_DIR_NAME) src_control_dir_file = os.path.join(src_dir, _CONTROL_DIR_FILE) dst_path = os.path.join(dst_dir, src_dir[prefix_len:]) dst_control_dir = os.path.join(dst_path, _CONTROL_DIR_NAME) scan_dir = None if not os.path.exists(dst_control_dir): fs.mkdir_safe(dst_control_dir) if rec: rec.write('%s\n' % os.path.join(dst_control_dir, '')) if pkg_resources.resource_isdir(package_name, src_control_dir): _install(package, src_control_dir, dst_dir, params, prefix_len=prefix_len, rec=rec) elif pkg_resources.resource_exists(package_name, src_control_dir_file): _LOGGER.info('Expand control dir: %s => %s', src_control_dir_file, dst_control_dir) svscan_conf_file = pkg_resources.resource_string( package_name, src_control_dir_file ) if svscan_conf_file: svscan_conf = yaml.load(svscan_conf_file.decode('utf8')) else: svscan_conf = {} scan_dir = supervisor.create_scan_dir( dst_path, svscan_conf.get('finish_timeout', 0), wait_cgroups=svscan_conf.get('wait_cgroups', None), kill_svc=svscan_conf.get('kill_svc', None) ) scan_dir.write() if not scan_dir: scan_dir = supervisor.ScanDir(dst_path) _install_services( scan_dir, package, src_dir, dst_dir, params, prefix_len=prefix_len, rec=rec )
def _install(package, src_dir, dst_dir, params, prefix_len=None, rec=None): """Interpolate source directory into target directory with params. """ package_name = package.__name__ _LOGGER.info('Installing package: %s %s %s', package_name, src_dir, dst_dir) contents = pkg_resources.resource_listdir(package_name, src_dir) if prefix_len is None: prefix_len = len(src_dir) + 1 for item in contents: resource_path = os.path.join(src_dir, item) dst_path = os.path.join(dst_dir, resource_path[prefix_len:]) if pkg_resources.resource_isdir(package_name, os.path.join(src_dir, item)): fs.mkdir_safe(dst_path) try: with io.open(os.path.join(dst_path, '.owner'), 'r') as f: owner = str(f.read().strip()) _LOGGER.info('Setting owner: %r - %r', dst_path, owner) owner_pw = pwd.getpwnam(owner) os.chown(dst_path, owner_pw.pw_uid, owner_pw.pw_gid) except (IOError, OSError) as err: if err.errno != errno.ENOENT: raise if rec: rec.write('%s\n' % os.path.join(dst_path, '')) install_fn = _install # Test if is a scan dir first if _is_scan_dir(package, os.path.join(src_dir, item), dst_path): _LOGGER.info('Scan dir found: %s => %s', resource_path, dst_path) install_fn = _install_scan_dir install_fn(package, os.path.join(src_dir, item), dst_dir, params, prefix_len=prefix_len, rec=rec) else: if resource_path.endswith('.swp'): continue resource_str = pkg_resources.resource_string( package_name, resource_path) if rec: rec.write('%s\n' % dst_path) _update(dst_path, _render(resource_str.decode('utf8'), params))
def handle(self, plugin, path, **params): path = '/'.join(path) # First check for the file in htdocs try: if plugin != '' and \ resource_exists(plugin, 'htdocs/' + path) and \ resource_isdir(plugin, 'htdocs/' + path) is False: mimetype, __encoding = mimetypes.guess_type(path, strict=False) if mimetype is not None: cherrypy.response.headers['Content-Type'] = mimetype return resource_stream(plugin, 'htdocs/' + path) except Exception as __error: pass menu = [] for observer in self.observers: arr = observer.getMenuItems() if isinstance(arr, list): menu.extend(arr) template = None templateDirs = [] response = None request = WebRequest(cherrypy.request) for observer in self.observers: if not observer.matchRequest(plugin, path): continue requireAuth = observer.requireAuthentication(plugin, path) if requireAuth != False: WebRequestHandler(self.context).isUrlAuthorized(request) response = observer.handleRequest(plugin, path, params, request=request) templateDirs = observer.getTemplatesDirs() break if response is None: raise cherrypy.NotFound() if isinstance(response, WebResponseRedirect): if response.url[:4] == 'http': raise cherrypy.HTTPRedirect(response.url) raise cherrypy.HTTPRedirect( '/%s%s%s' % (plugin, '' if response.url[0] == '/' else '/', response.url)) elif isinstance(response, WebResponse): if isinstance(response, WebResponseHtml): response.setDirs(plugin, templateDirs) cherrypy.response.status = response.statusCode response.output(cherrypy.response) return response.data template, data = response if template is None: raise cherrypy.NotFound() tmpl = self.loadTemplate(template, templateDirs) data['menu'] = menu stream = tmpl.generate(title='TellStick ZNet', **data) return stream.render('html', doctype='html')
def test_pr_resources(self): import pkg_resources as pr # App ZIP pkg = "android1" names = ["subdir", "__init__.py", "a.txt", "b.so", "mod1.py"] self.assertCountEqual(names, pr.resource_listdir(pkg, "")) for name in names: with self.subTest(name=name): self.assertTrue(pr.resource_exists(pkg, name)) self.assertEqual(pr.resource_isdir(pkg, name), name == "subdir") self.assertFalse(pr.resource_exists(pkg, "nonexistent")) self.assertFalse(pr.resource_isdir(pkg, "nonexistent")) self.assertCountEqual(["c.txt"], pr.resource_listdir(pkg, "subdir")) self.assertTrue(pr.resource_exists(pkg, "subdir/c.txt")) self.assertFalse(pr.resource_isdir(pkg, "subdir/c.txt")) self.assertFalse(pr.resource_exists(pkg, "subdir/nonexistent.txt")) self.check_pr_resource(APP_ZIP, pkg, "__init__.py", b"# This package is") self.check_pr_resource(APP_ZIP, pkg, "a.txt", b"alpha\n") self.check_pr_resource(APP_ZIP, pkg, "b.so", b"bravo\n") self.check_pr_resource(APP_ZIP, pkg, "subdir/c.txt", b"charlie\n") # Requirements ZIP self.reset_package("murmurhash") self.assertCountEqual([ "include", "tests", "__init__.pxd", "__init__.pyc", "about.pyc", "mrmr.pxd", "mrmr.pyx", "mrmr.so" ], pr.resource_listdir("murmurhash", "")) self.assertCountEqual(["MurmurHash2.h", "MurmurHash3.h"], pr.resource_listdir("murmurhash", "include/murmurhash")) self.check_pr_resource(REQS_COMMON_ZIP, "murmurhash", "__init__.pyc", MAGIC_NUMBER) self.check_pr_resource(REQS_COMMON_ZIP, "murmurhash", "mrmr.pxd", b"from libc.stdint") self.check_pr_resource(REQS_ABI_ZIP, "murmurhash", "mrmr.so", b"\x7fELF")
def _recursive_overwrite(pkg, src, dest): if pkg_resources.resource_isdir(pkg, src): if not os.path.isdir(dest): os.makedirs(dest) contents = pkg_resources.resource_listdir(pkg, src) for item in contents: _recursive_overwrite(pkg, os.path.join(src, item), os.path.join(dest, item)) else: filename = pkg_resources.resource_filename(pkg, src) shutil.copyfile(filename, dest)
def _resource_tree(package, resource): result = [] for name in pkg_resources.resource_listdir(package, resource): path = os.path.join(resource, name) if pkg_resources.resource_isdir(package, path): result.append(name) subtree = _resource_tree(package, path) result += [os.path.join(name, x) for x in subtree] else: result.append(name) return result
def _walk_pkg_resources(package, root): paths = pkg_resources.resource_listdir(package, root) for path in paths: dir_paths = [ path for path in paths if pkg_resources.resource_isdir(package, os.path.join(root, path)) ] yield root, list(set(paths).difference(dir_paths)) for dir_path in dir_paths: yield from _walk_pkg_resources(package, os.path.join(root, dir_path))
def copy_dir(source, dest, variables, out_=sys.stdout, i=0): """ Copies the ``source`` directory to the ``dest`` directory, where ``source`` is some tuple representing an installed package and a subdirectory in the package, e.g., ('pecan', os.path.join('scaffolds', 'base')) ('pecan_extension', os.path.join('scaffolds', 'scaffold_name')) ``variables``: A dictionary of variables to use in any substitutions. Substitution is performed via ``string.Template``. ``out_``: File object to write to (default is sys.stdout). """ def out(msg): out_.write('%s%s' % (' ' * (i * 2), msg)) out_.write('\n') out_.flush() names = sorted(pkg_resources.resource_listdir(source[0], source[1])) if not os.path.exists(dest): out('Creating %s' % dest) makedirs(dest) else: out('%s already exists' % dest) return for name in names: full = '/'.join([source[1], name]) dest_full = os.path.join(dest, substitute_filename(name, variables)) sub_file = False if dest_full.endswith('_tmpl'): dest_full = dest_full[:-5] sub_file = True if pkg_resources.resource_isdir(source[0], full): out('Recursing into %s' % os.path.basename(full)) copy_dir((source[0], full), dest_full, variables, out_, i + 1) continue else: content = pkg_resources.resource_string(source[0], full) if sub_file: content = render_template(content, variables) if content is None: continue # pragma: no cover out('Copying %s to %s' % (full, dest_full)) f = open(dest_full, 'wb') f.write(content) f.close()
def is_dir(self, path): """Is the resource a directory Args: path (str): resource location """ try: self.exists(path, True) return resource_isdir(self._module, path) except ImportError: raise ImportError(self._module) from None
def _list_schema_names(): package_name = 'sme_finance_application_schema' python_resource_re = re.compile(r'.*\.py[cod]?$') for filename in resource_listdir(package_name, ''): # type: str if python_resource_re.match(filename): continue elif filename.startswith(('_', '.')): # we care not about hidden files continue elif resource_isdir(package_name, filename): # we have no nesting continue yield filename
def get_filename(path, always=True, only_files=False, source=SOURCE_ANY): """ Find the file at the given ``path`` and return the absolute path. If ``only_files`` is True, paths will be checked to ensure that they're not directories. This function first searches in the active profile's directory. If the file is not found there, and configuration is available, it attempts to use `pkg_resources <http://packages.python.org/distribute/pkg_resources.html>`_ to locate the file, otherwise it will search the root directory. If a file cannot be found, and ``always`` is True, a path to a hypothetical file in the active profile's directory will be returned. If ``always`` is True and ``only_files`` is also True, and the path in the active profile's directory exists and is a directory, returns ``None``. Otherwise, if a file cannot be found, returns ``None``. **Note:** You should use this function as little as possible, as it may require a file to be temporarily written to a cache if the file is found with ``pkg_resources``. """ if os.path.isabs(path): if only_files and os.path.isdir(path): return None if always or os.path.exists(path): return path # Not an absolute path, so process it. ensure_paths() p_path = os.path.abspath(os.path.join(profile_path, path)) if source & SOURCE_PROFILE: if os.path.exists(p_path) and (not only_files or (only_files and os.path.isfile(p_path))): return p_path if source & SOURCE_PKG_RESOURCES: if (package and pr and pr.resource_exists(package, path) and (not only_files or (only_files and not pr.resource_isdir(package, path)))): return pr.resource_filename(package, path) if source & SOURCE_ROOT: r_path = os.path.abspath(os.path.join(root_path, path)) if os.path.exists(r_path) and (not only_files or (only_files and os.path.isfile(r_path))): return r_path if always and (not only_files or (only_files and not os.path.isdir(p_path))): return p_path
def get_type(self, config_path: str) -> ObjectType: full_path = self.concat(self.path, config_path) module_name, resource_name = PackageConfigSource._split_module_and_resource( full_path) if resource_exists(module_name, resource_name): if resource_isdir(module_name, resource_name): return ObjectType.GROUP else: return ObjectType.CONFIG else: return ObjectType.NOT_FOUND
async def process_resources(self, watcher=None): self.processed_resources = {} self.urls = {} expanded_resources = [] for resource in self.resources: if isinstance(resource, self.ResourceDir): for name in pkg_resources.resource_listdir( self.package, resource.resource_name): resource_name = f'{resource.resource_name}/{name}' if not pkg_resources.resource_isdir( self.package, resource_name): resources_key = (resource.route_name, name) expanded_resources.append( self.Resource(resource_name, resource.route_name, resources_key, resource_name, {'path': name}, None, None, resource.use_hased_url, resource.cache_control, resource.response_kwarg)) else: expanded_resources.append(resource) for resource in expanded_resources: response_kwarg = copy(resource.response_kwarg) body, hash = await self.get_static_processed_resource( resource.resource_name, resource.body_processor, resource.body_loader, watcher) if 'content_type' not in response_kwarg: _, extension = os.path.splitext(resource.resource_name) with suppress(KeyError): response_kwarg['content_type'] = mimetypes.types_map[ extension] response_kwarg['body'] = body if resource.use_hased_url: url = self.app.router[resource.route_name].url_for( **resource.url_for_kwargs).with_query({'hash': hash}) else: url = self.app.router[resource.route_name].url_for( **resource.url_for_kwargs) self.processed_resources[ resource.resources_key] = self.ResourceProcessed( resource.use_hased_url, resource.cache_control, url, hash, response_kwarg) self.urls[resource.urls_key] = url # if logger.isEnabledFor(logging.DEBUG): # import pprint # logger.debug('Urls: \n{}'.format(pprint.pformat(self.urls))) await self.resources_processed(self)
def browser_folder( cls, src_module, src_folder, searched_folder="templates/", context=None ): """Return the set of all filenames in the `src_folder` (relative to the `src_module` Python module). If the filename ends with '_tpl', then this suffix will be removed (for compatibility reasons). If the filename ends with '_inc', then it is ignored. Returned filenames are relative to this base folder. You can use formatted filenames: given the `{'var': 'myproject'}` context, the `'{var}/models.py'` filename will be returned as `'myproject/models.py'`. >>> m = TemplatedBaseCommand.browser_folder >>> result = m('djangofloor', 'djangofloor/test', context={'var': 'my_project'}) >>> for x in sorted(result.items()): ... print(x) ('subtest/index.html', 'djangofloor/test/subtest/index.html_tpl') ('subtest/my_project.txt', 'djangofloor/test/subtest/{var}.txt') ('text.txt', 'djangofloor/test/text.txt') """ formatter = string.Formatter() result = {} template_root_len = len(searched_folder) filename_root_len = len(src_folder) if not src_folder.endswith("/"): filename_root_len += 1 searched_base = searched_folder + src_folder if pkg_resources.resource_isdir(src_module, searched_base): for (root, dirnames, filenames) in walk(src_module, searched_base): for filename in filenames: if filename.endswith(cls.ignored_suffix): continue src_path = os.path.join(root, filename)[template_root_len:] if src_path.endswith(cls.transparent_suffix): dst_path = src_path[ filename_root_len : -len(cls.transparent_suffix) ] else: dst_path = src_path[filename_root_len:] if context and "{" in dst_path: format_values = {} for ( literal_text, field_name, format_spec, conversion, ) in formatter.parse(dst_path): if field_name is not None: format_values[field_name] = context[field_name] if format_values: dst_path = formatter.format(dst_path, **format_values) result[dst_path] = src_path return result
def test_pkg_resources_resources(self): import pkg_resources as pr self.assertTrue(pr.resource_exists(__package__, "test_android.py")) self.assertTrue(pr.resource_exists(__package__, "resources")) self.assertFalse(pr.resource_exists(__package__, "nonexistent")) self.assertTrue(pr.resource_exists(__package__, "resources/a.txt")) self.assertFalse( pr.resource_exists(__package__, "resources/nonexistent.txt")) self.assertFalse(pr.resource_isdir(__package__, "test_android.py")) self.assertTrue(pr.resource_isdir(__package__, "resources")) self.assertFalse(pr.resource_isdir(__package__, "nonexistent")) self.assertCountEqual(["a.txt", "b.so", "subdir"], pr.resource_listdir(__package__, "resources")) self.assertEqual(b"alpha\n", pr.resource_string(__package__, "resources/a.txt")) self.assertEqual(b"bravo\n", pr.resource_string(__package__, "resources/b.so")) self.assertCountEqual(["c.txt"], pr.resource_listdir(__package__, "resources/subdir")) self.assertEqual( b"charlie\n", pr.resource_string(__package__, "resources/subdir/c.txt")) # App ZIP a_filename = pr.resource_filename(__package__, "resources/a.txt") self.assertEqual(asset_path(APP_ZIP, "chaquopy/test/resources/a.txt"), a_filename) with open(a_filename) as a_file: self.assertEqual("alpha\n", a_file.read()) # Requirements ZIP cacert_filename = pr.resource_filename("certifi", "cacert.pem") self.assertEqual(asset_path(REQS_COMMON_ZIP, "certifi/cacert.pem"), cacert_filename) with open(cacert_filename) as cacert_file: self.assertTrue(cacert_file.read().startswith( "\n# Issuer: CN=GlobalSign Root CA O=GlobalSign nv-sa OU=Root CA" ))
def _iter(plugin, module): ''' Iterate over migrations for a given plugin module Yield tuples in the form (plugin_name, module_name, filename) ''' module_name = module if isinstance(module, str) else module.__name__ if not resource_isdir(module_name, 'migrations'): return for filename in resource_listdir(module_name, 'migrations'): if filename.endswith('.py') and not filename.startswith('__'): yield Migration(plugin, filename, module_name)
def _copy_static_from_resources(self, src, dst): """Copy static resources from the distribution package. :param src: the source package directory :param dst: the destination on the file system """ logger.info('copy: {} -> {}'.format(src, dst)) if pkg_resources.resource_isdir(__name__, src): logger.debug('mkdir: {}'.format(dst)) os.makedirs(dst, exist_ok=True) for res in pkg_resources.resource_listdir(__name__, src): dst_res = os.path.join(dst, os.path.basename(res)) res = '/'.join([src, res]) if pkg_resources.resource_isdir(__name__, res): self._copy_static_from_resources(res, dst_res) else: in_stream = pkg_resources.resource_stream(__name__, res) logger.debug('copy file: {} -> {}'.format(res, dst_res)) with open(dst_res, 'wb') as fout: shutil.copyfileobj(in_stream, fout)
def resources_for_test_config(test_config): resources = {} for key in [constants.CONFIG_BUNDLES_KEY, 'apps', 'libs', 'services']: key_path = 'test_configs/{}/{}'.format(test_config, key) if resource_isdir(__name__, key_path): resources[key] = { resource_name: resource_string(__name__, '{}/{}'.format(key_path, resource_name)) for resource_name in resource_listdir(__name__, key_path) } return resources
def extractdir(self, resourcedir, target, suffixes=None): """Extract all file resources contained in the specified resource directory to the target directory. Searches all loadpaths and optionally the Resources API for any file contained within. This means the target dir may end up with eg. one file from a high-priority path and other files from the system dirs/resources. This in turns makes it easy to just override a single file in a larger set of resource files. Even if the resourcedir might contain resources in subdirectories (eg "source/sub/dir/resource.xml"), the extraction will be to the top-level target directory (eg "target/resource.xml"). """ if not suffixes: suffixes = [] extracted = set() for path in self.loadpath: if resourcedir and resourcedir != ".": path = path + os.sep + resourcedir if not os.path.exists(path): continue # for f in os.listdir(path): for f in util.list_dirs(path, suffixes): f = f[len(path) + 1:] basef = os.path.basename(f) src = os.sep.join([path, f]) dest = os.sep.join([target, basef]) if dest not in extracted and os.path.isfile(src): util.ensure_dir(dest) shutil.copy2(src, dest) extracted.add(dest) if self.use_pkg_resources: self._check_module_path() path = self.resourceprefix if resourcedir: path = path + os.sep + resourcedir for f in pkg_resources.resource_listdir(self.modulename, path): src = path + os.sep + f dest = target dest += os.sep + f if (dest not in extracted and not pkg_resources.resource_isdir( self.modulename, self.resourceprefix + os.sep + f)): util.ensure_dir(dest) with open(dest, "wb") as fp: readfp = pkg_resources.resource_stream( self.modulename, src) fp.write(readfp.read()) readfp.close() extracted.add(dest)
def image_resources(pkg=None, directory='resources'): if pkg is None: pkg = calling_package() pkg_dir = '%s.%s' % (pkg, directory) images = [] for i in resource_listdir(pkg, directory): fname = resource_filename(pkg_dir, i) if resource_isdir(pkg_dir, i): images.extend(image_resources(pkg_dir, i)) elif what(fname) is not None: images.append(fname) return images
def _walk_assets(directory): assets = pkg_resources.resource_listdir(__name__, directory) files, dirs = [], [] for f in assets: if pkg_resources.resource_isdir(__name__, os.path.join(directory, f)): dirs.append(f) else: files.append(f) yield directory, dirs, files for dir in dirs: for r, d, f in Doc._walk_assets(os.path.join(directory, dir)): yield r, d, f
def gen_resources(path): for entry in pkg_resources.resource_listdir(__name__, path): full_entry = SEP.join([path, entry]) if pkg_resources.resource_isdir(__name__, full_entry): yield True, entry, full_entry else: config_name, sep, ext = entry.rpartition('.') if ext is not None and ext.casefold() == FileExt.YAML.value: yield False, config_name, full_entry