def __call__(self, broker): ctx = _get_context(self.context, broker) root = ctx.root results = [] for pattern in self.patterns: pattern = ctx.locate_path(pattern) for path in sorted(glob(os.path.join(root, pattern.lstrip('/')))): if self.ignore_func(path) or os.path.isdir(path): continue try: results.append( self.kind(path[len(root):], root=root, ds=self, ctx=ctx)) except: log.debug(traceback.format_exc()) if results: if len(results) > self.max_files: raise ContentException( "Number of files returned [{0}] is over the {1} file limit, please refine " "the specs file pattern to narrow down results".format( len(results), self.max_files)) return results raise ContentException("[%s] didn't match." % ', '.join(self.patterns))
def validate(self): if not blacklist.allow_file("/" + self.relative_path): raise dr.SkipComponent() if not os.path.exists(self.path): raise ContentException("%s does not exist." % self.path) if not os.access(self.path, os.R_OK): raise ContentException("Cannot access %s" % self.path)
def __call__(self, broker): source = broker[self.provider] ctx = broker[self.context] if not isinstance(source, (str, tuple)): raise ContentException("The provider can only be a single string or a tuple of strings, but got '%s'." % source) try: self.cmd = self.cmd % source return CommandOutputProvider(self.cmd, ctx, split=self.split, keep_rc=self.keep_rc, ds=self, timeout=self.timeout, inherit_env=self.inherit_env) except: log.debug(traceback.format_exc()) raise ContentException("No results found for [%s]" % self.cmd)
def validate(self): if not blacklist.allow_file("/" + self.relative_path): raise dr.SkipComponent() if not os.path.exists(self.path): raise ContentException("%s does not exist." % self.path) resolved = os.path.realpath(self.path) if not resolved.startswith(os.path.realpath(self.root)): msg = "Relative path points outside the root: %s -> %s." raise Exception(msg % (self.path, resolved)) if not os.access(self.path, os.R_OK): raise ContentException("Cannot access %s" % self.path)
def _stream(self): """ Returns a generator of lines instead of a list of lines. """ if self._exception: raise self._exception try: if self._content: yield self._content raise StopIteration filters = get_filters(self.ds) if self.ds else None if filters: grep = ["grep", "-F", "\n".join(filters)] with self.ctx.connect(self.cmd, grep, env=SAFE_ENV, timeout=self.timeout) as s: yield s else: with self.ctx.stream(self.cmd, env=SAFE_ENV, timeout=self.timeout) as s: yield s except StopIteration: raise except Exception as ex: self._exception = ex raise ContentException(str(ex))
def validate(self): if not blacklist.allow_command(self.cmd): log.warning("WARNING: Skipping command %s", self.cmd) raise dr.SkipComponent() if not which(shlex.split(self.cmd)[0], env=self.create_env()): raise ContentException("Couldn't execute: %s" % self.cmd)
def _stream(self): """ Returns a generator of lines instead of a list of lines. """ if self._exception: raise self._exception try: if self._content: yield self._content else: args = self.create_args() if args: with streams.connect(*args, env=SAFE_ENV) as s: yield s else: if six.PY3: with open(self.path, "r", encoding="utf-8", errors="surrogateescape") as f: yield f else: with codecs.open(self.path, "r", encoding="utf-8", errors="surrogateescape") as f: yield f except StopIteration: raise except Exception as ex: self._exception = ex raise ContentException(str(ex))
def inner(broker): result = [] source = broker[provider] ctx = broker[context] if isinstance(source, ContentProvider): source = source.content if not isinstance(source, (list, set)): source = [source] for e in source: try: the_cmd = cmd % e rc = None raw = ctx.shell_out(the_cmd, split=split, keep_rc=keep_rc, timeout=timeout) if keep_rc: rc, output = raw else: output = raw result.append( CommandOutputProvider(the_cmd, ctx, args=e, content=output, rc=rc, split=split, keep_rc=keep_rc)) except: log.debug(traceback.format_exc()) if result: return result raise ContentException("No results found for [%s]" % cmd)
def __call__(self, broker): result = [] source = broker[self.provider] ctx = broker[self.context] if isinstance(source, ContentProvider): source = source.content if not isinstance(source, (list, set)): source = [source] for e in source: try: the_cmd = self.cmd % e cop = CommandOutputProvider(the_cmd, ctx, args=e, split=self.split, keep_rc=self.keep_rc, ds=self, timeout=self.timeout, inherit_env=self.inherit_env) result.append(cop) except: log.debug(traceback.format_exc()) if result: return result raise ContentException("No results found for [%s]" % self.cmd)
def __call__(self, broker): ctx = _get_context(self.context, broker) root = ctx.root for p in self.paths: try: return self.kind(ctx.locate_path(p), root=root, ds=self, ctx=ctx) except: pass raise ContentException("None of [%s] found." % ', '.join(self.paths))
def __call__(self, broker): ctx = _get_context(self.context, broker) p = os.path.join(ctx.root, self.path.lstrip('/')) p = ctx.locate_path(p) result = sorted(os.listdir(p)) if os.path.isdir(p) else sorted(glob(p)) if result: return [os.path.basename(r) for r in result if not self.ignore_func(r)] raise ContentException("Can't list %s or nothing there." % p)
def inner(broker): ctx = _get_context(context, FSRoots, broker) root = ctx.root for f in files: try: return kind(ctx.locate_path(f), root=root, ds=inner) except: pass raise ContentException("None of [%s] found." % ', '.join(files))
def inner(broker): ctx = _get_context(context, FSRoots, broker) p = os.path.join(ctx.root, path.lstrip('/')) p = ctx.locate_path(p) if os.path.isdir(p): return os.listdir(p) result = glob(p) if result: return [os.path.basename(r) for r in result] raise ContentException("Can't list %s or nothing there." % p)
def inner(broker): ctx = _get_context(context, FSRoots, broker) root = ctx.root results = [] for pattern in patterns: pattern = ctx.locate_path(pattern) for path in sorted(glob(os.path.join(root, pattern.lstrip('/')))): if ignore and re.search(ignore, path): continue try: results.append(kind(path[len(root):], root=root, ds=inner)) except: log.debug(traceback.format_exc()) if results: if len(results) > max_files: raise ContentException( "Number of files returned [{0}] is over the {1} file limit, please refine " "the specs file pattern to narrow down results".format( len(results), max_files)) return results raise ContentException("[%s] didn't match." % ', '.join(patterns))
def inner(broker): ctx = _get_context(context, FSRoots, broker) root = ctx.root results = [] for pattern in patterns: pattern = ctx.locate_path(pattern) for path in glob(os.path.join(root, pattern.lstrip('/'))): if ignore and re.search(ignore, path): continue try: results.append(kind(path[len(root):], root=root, ds=inner)) except: log.debug(traceback.format_exc()) if results: return results raise ContentException("[%s] didn't match." % ', '.join(patterns))
def __call__(self, broker): result = [] source = broker[self.provider] ctx = broker[self.context] if isinstance(source, ContentProvider): source = source.content if not isinstance(source, (list, set)): source = [source] for e in source: try: the_cmd = self.cmd % e rc = None if self.split: filters = "\n".join(get_filters(self)) if filters: command = [shlex.split(the_cmd) ] + [["grep", "-F", filters]] raw = ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout) else: command = [shlex.split(the_cmd)] raw = ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout) if self.keep_rc: rc, output = raw else: output = raw result.append( CommandOutputProvider(the_cmd, ctx, args=e, content=output, rc=rc, split=self.split, keep_rc=self.keep_rc)) except: log.debug(traceback.format_exc()) if result: return result raise ContentException("No results found for [%s]" % self.cmd)
def _stream(self): """ Returns a generator of lines instead of a list of lines. """ if self._exception: raise self._exception try: if self._content: yield self._content else: args = self.create_args() with self.ctx.connect(*args, env=self.create_env(), timeout=self.timeout) as s: yield s except StopIteration: raise except Exception as ex: self._exception = ex raise ContentException(str(ex))
def load(self): filters = False if self.ds: filters = "\n".join(get_filters(self.ds)) if filters: cmd = "/bin/grep -F '{0}' {1}".format(filters, self.path) rc, out = subproc.call(cmd.encode("utf-8"), shell=False, keep_rc=True) if rc == 0 and out != '': results = out.splitlines() else: return [] else: with open(self.path, "rU") as f: results = [l.rstrip("\n") for l in f] if not self.validate_lines(results): first = results[0] if results else "<no content>" raise ContentException(self.relative_path + ": " + first) return results
def inner(broker): result = [] source = broker[provider] ctx = _get_context(context, FSRoots, broker) root = ctx.root if isinstance(source, ContentProvider): source = source.content if not isinstance(source, (list, set)): source = [source] for e in source: pattern = ctx.locate_path(path % e) for p in glob(os.path.join(root, pattern.lstrip('/'))): if ignore and re.search(ignore, p): continue try: result.append(kind(p[len(root):], root=root, ds=inner)) except: log.debug(traceback.format_exc()) if result: return result raise ContentException("No results found for [%s]" % path)
def __call__(self, broker): result = [] source = broker[self.provider] ctx = _get_context(self.context, broker) root = ctx.root if isinstance(source, ContentProvider): source = source.content if not isinstance(source, (list, set)): source = [source] for e in source: pattern = ctx.locate_path(self.path % e) for p in glob(os.path.join(root, pattern.lstrip('/'))): if self.ignore_func(p) or os.path.isdir(p): continue try: result.append(self.kind(p[len(root):], root=root, ds=self, ctx=ctx)) except: log.debug(traceback.format_exc()) if result: return result raise ContentException("No results found for [%s]" % self.path)
def _stream(self): """ Returns a generator of lines instead of a list of lines. """ if self._exception: raise self._exception try: if self._content: yield self._content else: args = self.create_args() if args: with streams.connect(*args, env=SAFE_ENV) as s: yield s else: with open(self.path, "rU") as f: # universal newlines yield f except StopIteration: raise except Exception as ex: self._exception = ex raise ContentException(str(ex))
def _stream(self): """ Returns a generator of lines instead of a list of lines. """ if self._exception: raise self._exception try: if self._content: yield self._content raise StopIteration filters = get_filters(self.ds) if self.ds else None if filters: cmd = ["grep", "-F", "\n".join(filters), self.path] with streams.stream(cmd, env=SAFE_ENV) as s: yield s else: with open(self.path, "rU") as f: # universal newlines yield f except StopIteration: raise except Exception as ex: self._exception = ex raise ContentException(str(ex))
def validate(self): if not blacklist.allow_command(self.cmd): raise dr.SkipComponent() if not which(shlex.split(self.cmd)[0]): raise ContentException("Couldn't execute: %s" % self.cmd)