def __call__(self, broker):
ctx = _get_context(self.context, broker)
root = ctx.root
results = []
for pattern in self.patterns:
pattern = ctx.locate_path(pattern)
for path in sorted(glob(os.path.join(root, pattern.lstrip('/')))):
if self.ignore_func(path) or os.path.isdir(path):
continue
try:
results.append(
self.kind(path[len(root):],
root=root,
ds=self,
ctx=ctx))
except:
log.debug(traceback.format_exc())
if results:
if len(results) > self.max_files:
raise ContentException(
"Number of files returned [{0}] is over the {1} file limit, please refine "
"the specs file pattern to narrow down results".format(
len(results), self.max_files))
return results
raise ContentException("[%s] didn't match." % ', '.join(self.patterns))
def validate(self):
if not blacklist.allow_file("/" + self.relative_path):
raise dr.SkipComponent()
if not os.path.exists(self.path):
raise ContentException("%s does not exist." % self.path)
if not os.access(self.path, os.R_OK):
raise ContentException("Cannot access %s" % self.path)
def __call__(self, broker):
source = broker[self.provider]
ctx = broker[self.context]
if not isinstance(source, (str, tuple)):
raise ContentException("The provider can only be a single string or a tuple of strings, but got '%s'." % source)
try:
self.cmd = self.cmd % source
return CommandOutputProvider(self.cmd, ctx, split=self.split,
keep_rc=self.keep_rc, ds=self, timeout=self.timeout, inherit_env=self.inherit_env)
except:
log.debug(traceback.format_exc())
raise ContentException("No results found for [%s]" % self.cmd)
def validate(self):
if not blacklist.allow_file("/" + self.relative_path):
raise dr.SkipComponent()
if not os.path.exists(self.path):
raise ContentException("%s does not exist." % self.path)
resolved = os.path.realpath(self.path)
if not resolved.startswith(os.path.realpath(self.root)):
msg = "Relative path points outside the root: %s -> %s."
raise Exception(msg % (self.path, resolved))
if not os.access(self.path, os.R_OK):
raise ContentException("Cannot access %s" % self.path)
def _stream(self):
"""
Returns a generator of lines instead of a list of lines.
"""
if self._exception:
raise self._exception
try:
if self._content:
yield self._content
raise StopIteration
filters = get_filters(self.ds) if self.ds else None
if filters:
grep = ["grep", "-F", "\n".join(filters)]
with self.ctx.connect(self.cmd,
grep,
env=SAFE_ENV,
timeout=self.timeout) as s:
yield s
else:
with self.ctx.stream(self.cmd,
env=SAFE_ENV,
timeout=self.timeout) as s:
yield s
except StopIteration:
raise
except Exception as ex:
self._exception = ex
raise ContentException(str(ex))
def validate(self):
if not blacklist.allow_command(self.cmd):
log.warning("WARNING: Skipping command %s", self.cmd)
raise dr.SkipComponent()
if not which(shlex.split(self.cmd)[0], env=self.create_env()):
raise ContentException("Couldn't execute: %s" % self.cmd)
def _stream(self):
"""
Returns a generator of lines instead of a list of lines.
"""
if self._exception:
raise self._exception
try:
if self._content:
yield self._content
else:
args = self.create_args()
if args:
with streams.connect(*args, env=SAFE_ENV) as s:
yield s
else:
if six.PY3:
with open(self.path, "r", encoding="utf-8", errors="surrogateescape") as f:
yield f
else:
with codecs.open(self.path, "r", encoding="utf-8", errors="surrogateescape") as f:
yield f
except StopIteration:
raise
except Exception as ex:
self._exception = ex
raise ContentException(str(ex))
def inner(broker):
result = []
source = broker[provider]
ctx = broker[context]
if isinstance(source, ContentProvider):
source = source.content
if not isinstance(source, (list, set)):
source = [source]
for e in source:
try:
the_cmd = cmd % e
rc = None
raw = ctx.shell_out(the_cmd,
split=split,
keep_rc=keep_rc,
timeout=timeout)
if keep_rc:
rc, output = raw
else:
output = raw
result.append(
CommandOutputProvider(the_cmd,
ctx,
args=e,
content=output,
rc=rc,
split=split,
keep_rc=keep_rc))
except:
log.debug(traceback.format_exc())
if result:
return result
raise ContentException("No results found for [%s]" % cmd)
def __call__(self, broker):
result = []
source = broker[self.provider]
ctx = broker[self.context]
if isinstance(source, ContentProvider):
source = source.content
if not isinstance(source, (list, set)):
source = [source]
for e in source:
try:
the_cmd = self.cmd % e
cop = CommandOutputProvider(the_cmd,
ctx,
args=e,
split=self.split,
keep_rc=self.keep_rc,
ds=self,
timeout=self.timeout,
inherit_env=self.inherit_env)
result.append(cop)
except:
log.debug(traceback.format_exc())
if result:
return result
raise ContentException("No results found for [%s]" % self.cmd)
def __call__(self, broker):
ctx = _get_context(self.context, broker)
root = ctx.root
for p in self.paths:
try:
return self.kind(ctx.locate_path(p), root=root, ds=self, ctx=ctx)
except:
pass
raise ContentException("None of [%s] found." % ', '.join(self.paths))
def __call__(self, broker):
ctx = _get_context(self.context, broker)
p = os.path.join(ctx.root, self.path.lstrip('/'))
p = ctx.locate_path(p)
result = sorted(os.listdir(p)) if os.path.isdir(p) else sorted(glob(p))
if result:
return [os.path.basename(r) for r in result if not self.ignore_func(r)]
raise ContentException("Can't list %s or nothing there." % p)
def inner(broker):
ctx = _get_context(context, FSRoots, broker)
root = ctx.root
for f in files:
try:
return kind(ctx.locate_path(f), root=root, ds=inner)
except:
pass
raise ContentException("None of [%s] found." % ', '.join(files))
def inner(broker):
ctx = _get_context(context, FSRoots, broker)
p = os.path.join(ctx.root, path.lstrip('/'))
p = ctx.locate_path(p)
if os.path.isdir(p):
return os.listdir(p)
result = glob(p)
if result:
return [os.path.basename(r) for r in result]
raise ContentException("Can't list %s or nothing there." % p)
def inner(broker):
ctx = _get_context(context, FSRoots, broker)
root = ctx.root
results = []
for pattern in patterns:
pattern = ctx.locate_path(pattern)
for path in sorted(glob(os.path.join(root, pattern.lstrip('/')))):
if ignore and re.search(ignore, path):
continue
try:
results.append(kind(path[len(root):], root=root, ds=inner))
except:
log.debug(traceback.format_exc())
if results:
if len(results) > max_files:
raise ContentException(
"Number of files returned [{0}] is over the {1} file limit, please refine "
"the specs file pattern to narrow down results".format(
len(results), max_files))
return results
raise ContentException("[%s] didn't match." % ', '.join(patterns))
def inner(broker):
ctx = _get_context(context, FSRoots, broker)
root = ctx.root
results = []
for pattern in patterns:
pattern = ctx.locate_path(pattern)
for path in glob(os.path.join(root, pattern.lstrip('/'))):
if ignore and re.search(ignore, path):
continue
try:
results.append(kind(path[len(root):], root=root, ds=inner))
except:
log.debug(traceback.format_exc())
if results:
return results
raise ContentException("[%s] didn't match." % ', '.join(patterns))
def __call__(self, broker):
result = []
source = broker[self.provider]
ctx = broker[self.context]
if isinstance(source, ContentProvider):
source = source.content
if not isinstance(source, (list, set)):
source = [source]
for e in source:
try:
the_cmd = self.cmd % e
rc = None
if self.split:
filters = "\n".join(get_filters(self))
if filters:
command = [shlex.split(the_cmd)
] + [["grep", "-F", filters]]
raw = ctx.shell_out(command,
split=self.split,
keep_rc=self.keep_rc,
timeout=self.timeout)
else:
command = [shlex.split(the_cmd)]
raw = ctx.shell_out(command,
split=self.split,
keep_rc=self.keep_rc,
timeout=self.timeout)
if self.keep_rc:
rc, output = raw
else:
output = raw
result.append(
CommandOutputProvider(the_cmd,
ctx,
args=e,
content=output,
rc=rc,
split=self.split,
keep_rc=self.keep_rc))
except:
log.debug(traceback.format_exc())
if result:
return result
raise ContentException("No results found for [%s]" % self.cmd)
def _stream(self):
"""
Returns a generator of lines instead of a list of lines.
"""
if self._exception:
raise self._exception
try:
if self._content:
yield self._content
else:
args = self.create_args()
with self.ctx.connect(*args, env=self.create_env(), timeout=self.timeout) as s:
yield s
except StopIteration:
raise
except Exception as ex:
self._exception = ex
raise ContentException(str(ex))
def load(self):
filters = False
if self.ds:
filters = "\n".join(get_filters(self.ds))
if filters:
cmd = "/bin/grep -F '{0}' {1}".format(filters, self.path)
rc, out = subproc.call(cmd.encode("utf-8"), shell=False, keep_rc=True)
if rc == 0 and out != '':
results = out.splitlines()
else:
return []
else:
with open(self.path, "rU") as f:
results = [l.rstrip("\n") for l in f]
if not self.validate_lines(results):
first = results[0] if results else "<no content>"
raise ContentException(self.relative_path + ": " + first)
return results
def inner(broker):
result = []
source = broker[provider]
ctx = _get_context(context, FSRoots, broker)
root = ctx.root
if isinstance(source, ContentProvider):
source = source.content
if not isinstance(source, (list, set)):
source = [source]
for e in source:
pattern = ctx.locate_path(path % e)
for p in glob(os.path.join(root, pattern.lstrip('/'))):
if ignore and re.search(ignore, p):
continue
try:
result.append(kind(p[len(root):], root=root, ds=inner))
except:
log.debug(traceback.format_exc())
if result:
return result
raise ContentException("No results found for [%s]" % path)
def __call__(self, broker):
result = []
source = broker[self.provider]
ctx = _get_context(self.context, broker)
root = ctx.root
if isinstance(source, ContentProvider):
source = source.content
if not isinstance(source, (list, set)):
source = [source]
for e in source:
pattern = ctx.locate_path(self.path % e)
for p in glob(os.path.join(root, pattern.lstrip('/'))):
if self.ignore_func(p) or os.path.isdir(p):
continue
try:
result.append(self.kind(p[len(root):], root=root, ds=self, ctx=ctx))
except:
log.debug(traceback.format_exc())
if result:
return result
raise ContentException("No results found for [%s]" % self.path)
def _stream(self):
"""
Returns a generator of lines instead of a list of lines.
"""
if self._exception:
raise self._exception
try:
if self._content:
yield self._content
else:
args = self.create_args()
if args:
with streams.connect(*args, env=SAFE_ENV) as s:
yield s
else:
with open(self.path, "rU") as f: # universal newlines
yield f
except StopIteration:
raise
except Exception as ex:
self._exception = ex
raise ContentException(str(ex))
def _stream(self):
"""
Returns a generator of lines instead of a list of lines.
"""
if self._exception:
raise self._exception
try:
if self._content:
yield self._content
raise StopIteration
filters = get_filters(self.ds) if self.ds else None
if filters:
cmd = ["grep", "-F", "\n".join(filters), self.path]
with streams.stream(cmd, env=SAFE_ENV) as s:
yield s
else:
with open(self.path, "rU") as f: # universal newlines
yield f
except StopIteration:
raise
except Exception as ex:
self._exception = ex
raise ContentException(str(ex))
def validate(self):
if not blacklist.allow_command(self.cmd):
raise dr.SkipComponent()
if not which(shlex.split(self.cmd)[0]):
raise ContentException("Couldn't execute: %s" % self.cmd)
