def __init__(self, regexlist, always_dirs=True):
self.always_dirs = always_dirs
if is_string(regexlist):
regexlist = [regexlist]
self.regexlist = []
for regex in regexlist:
if is_string(regex):
regex = re.compile(regex)
self.regexlist.append(regex)
def test_is_string(self):
"""Tests for is_string function."""
for item in ['foo', u'foo', "hello world", """foo\nbar""", '']:
self.assertTrue(is_string(item))
for item in [1, None, ['foo'], ('foo', ), {'foo': 'bar'}]:
self.assertFalse(is_string(item))
if is_py3():
self.assertFalse(is_string(b'bytes_are_not_a_string'))
else:
# in Python 2, b'foo' is really just a regular string
self.assertTrue(is_string(b'foo'))
def test_getDetailsLogLevels(self):
"""
Test the getDetailsLogLevels selection logic
(and also the getAllExistingLoggers, getAllFancyloggers and
getAllNonFancyloggers function call)
"""
# logger names are unique
for fancy, func in [(False, fancylogger.getAllNonFancyloggers),
(True, fancylogger.getAllFancyloggers),
(None, fancylogger.getAllExistingLoggers)]:
self.assertEqual(
[name for name, _ in func()],
[name for name, _ in fancylogger.getDetailsLogLevels(fancy)],
"Test getDetailsLogLevels fancy %s and function %s" %
(fancy, func.__name__))
self.assertEqual([
name for name, _ in fancylogger.getAllFancyloggers()
], [
name for name, _ in fancylogger.getDetailsLogLevels()
], "Test getDetailsLogLevels default fancy True and function getAllFancyloggers"
)
res = fancylogger.getDetailsLogLevels(fancy=True)
self.assertTrue(
is_string(res[0][1]),
msg='getDetailsLogLevels returns loglevel names by default')
res = fancylogger.getDetailsLogLevels(fancy=True, numeric=True)
self.assertTrue(
isinstance(res[0][1], int),
msg='getDetailsLogLevels returns loglevel values with numeric=True'
)
def test_set_columns(self):
"""Test set_columns function."""
def reset_columns():
"""Reset environment to run another test case for set_columns."""
if 'COLUMNS' in os.environ:
del os.environ['COLUMNS']
reset_columns()
set_columns()
cols = os.environ.get('COLUMNS')
self.assertTrue(cols is None or is_string(cols))
reset_columns()
set_columns(cols=10)
self.assertEqual(
os.environ['COLUMNS'],
'10',
msg='env COLUMNS equals set_columns, no previous COLUMNS')
# $COLUMNS wins
set_columns(cols=99)
self.assertEqual(os.environ['COLUMNS'],
'10',
msg='env COLUMNS equals previous COLUMNS')
reset_columns()
set_columns(cols=99)
self.assertEqual(
os.environ['COLUMNS'],
'99',
msg='env COLUMNS equals set_columns, no previous COLUMNS (99)')
reset_columns()
def add(self, items, tmpl_vals=None, allow_spaces=True):
"""
Add options/arguments to command
:param item: option/argument to add to command
:param tmpl_vals: template values for item
"""
if not isinstance(items, (list, tuple)):
items = [items]
for item in items:
if tmpl_vals:
item = item % tmpl_vals
if not is_string(item):
raise ValueError(
"Non-string item %s (type %s) being added to command %s" %
(item, type(item), self))
if not allow_spaces and ' ' in item:
raise ValueError(
"Found one or more spaces in item '%s' being added to command %s"
% (item, self))
super(CmdList, self).append(item)
def test_noshell_async_stdout_stdin(self):
# test with both regular strings and bytestrings (only matters w.r.t. Python 3 compatibility)
inputs = [
'foo',
b'foo',
"testing, 1, 2, 3",
b"testing, 1, 2, 3",
]
for inp in inputs:
self.mock_stderr(True)
self.mock_stdout(True)
ec, output = async_to_stdout('/bin/cat', input=inp)
stderr, stdout = self.get_stderr(), self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
# there should be no output to stderr
self.assertFalse(stderr)
# output is always string, not bytestring, so convert before comparison
# (only needed for Python 3)
if not is_string(inp):
inp = inp.decode(encoding='utf-8')
self.assertEqual(ec, 0)
self.assertEqual(output, inp)
self.assertEqual(output, stdout)
def getLevelInt(level_name):
"""Given a level name, return the int value"""
if not is_string(level_name):
raise TypeError('Provided name %s is not a string (type %s)' %
(level_name, type(level_name)))
level = logging.getLevelName(level_name)
if not isinstance(level, int):
raise MissingLevelName('Unknown loglevel name %s' % level_name)
return level
def getgrouplist(user, groupnames=True):
"""
Return a list of all groupid's for groups this user is in
This function is needed here because python's user database only contains local users, not remote users from e.g.
sssd
user can be either an integer (uid) or a string (username)
returns a list of groupnames
if groupames is false, returns a list of groupids (skip groupname lookups)
"""
libc = cdll.LoadLibrary(find_library('libc'))
getgrouplist = libc.getgrouplist
# max of 50 groups should be enough as first try
ngroups = 50
getgrouplist.argtypes = [
c_char_p, c_uint,
POINTER(c_uint * ngroups),
POINTER(c_int32)
]
getgrouplist.restype = c_int32
grouplist = (c_uint * ngroups)()
ngrouplist = c_int32(ngroups)
if is_string(user):
user = pwd.getpwnam(user)
else:
user = pwd.getpwuid(user)
# .encode() is required in Python 3, since we need to pass a bytestring to getgrouplist
user_name, user_gid = user.pw_name.encode(), user.pw_gid
ct = getgrouplist(user_name, user_gid, byref(grouplist), byref(ngrouplist))
# if a max of 50 groups was not enough, try again with exact given nr
if ct < 0:
getgrouplist.argtypes = [
c_char_p, c_uint,
POINTER(c_uint * int(ngrouplist.value)),
POINTER(c_int32)
]
grouplist = (c_uint * int(ngrouplist.value))()
ct = getgrouplist(user_name, user_gid, byref(grouplist),
byref(ngrouplist))
if ct < 0:
raise Exception(
"Could not find groups for %s: getgrouplist returned %s" %
(user, ct))
grouplist = [grouplist[i] for i in range(ct)]
if groupnames:
grouplist = [grp.getgrgid(i).gr_name for i in grouplist]
return grouplist
def setLogLevel(level):
"""
Set a global log level for all FancyLoggers
"""
if is_string(level):
level = getLevelInt(level)
logger = getLogger(fname=False, clsname=False)
logger.setLevel(level)
if _env_to_boolean('FANCYLOGGER_LOGLEVEL_DEBUG'):
print("FANCYLOGGER_LOGLEVEL_DEBUG", level, logging.getLevelName(level))
print("\n".join(logger.get_parent_info("FANCYLOGGER_LOGLEVEL_DEBUG")))
sys.stdout.flush()
def rsync_path(self, path):
""" start rsync session for given path and destination, returns true if successful """
if not path:
self.log.raiseException('Empty path given!')
return None
elif not is_string(path):
self.log.raiseException('Invalid path: %s !' % path)
return None
else:
code, output = self.attempt_run(path)
if output:
self.output_queue.put(output.encode())
return (code == 0)
def request(self, method, url, body, headers, content_type=None):
"""Low-level networking. All HTTP-method methods call this"""
# format headers
if headers is None:
headers = {}
if content_type is not None:
headers['Content-Type'] = content_type
if self.auth_header is not None:
headers['Authorization'] = self.auth_header
headers['User-Agent'] = self.user_agent
# censor contents of 'Authorization' part of header, to avoid leaking tokens or passwords in logs
secret_items = ['Authorization', 'X-Auth-Token']
headers_censored = self.censor_request(secret_items, headers)
if body and not is_string(body):
# censor contents of body to avoid leaking passwords
secret_items = ['password']
body_censored = self.censor_request(secret_items, body)
# serialize body in all cases
body = json.dumps(body)
else:
# assume serialized bodies are already clear of secrets
fancylogger.getLogger().debug(
"Request with pre-serialized body, will not censor secrets")
body_censored = body
fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method,
url, body_censored, headers_censored)
# TODO: in recent python: Context manager
conn = self.get_connection(method, url, body, headers)
status = conn.code
if method == self.HEAD:
pybody = conn.headers
else:
body = conn.read()
if is_py3():
body = body.decode('utf-8') # byte encoded response
try:
pybody = json.loads(body)
except ValueError:
pybody = body
fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
conn.close()
return status, pybody
def _make_shell_command(self):
"""Convert cmd into a list of command to be sent to popen, without a shell"""
if self.cmd is None:
self.log.raiseException("_make_shell_command: no cmd set.")
if is_string(self.cmd):
self._shellcmd = shlex.split(self.cmd)
elif isinstance(self.cmd, (
list,
tuple,
)):
self._shellcmd = self.cmd
else:
self.log.raiseException(
"Failed to convert cmd %s (type %s) into non shell command" %
(self.cmd, type(self.cmd)))
def process_answers(answers):
"""Construct list of newline-terminated answers (as strings)."""
if is_string(answers):
answers = [answers]
elif isinstance(answers, list):
# list is manipulated when answering matching question, so take a copy
answers = answers[:]
else:
msg_tmpl = "Invalid type for answer, not a string or list: %s (%s)"
self.log.raiseException(msg_tmpl % (type(answers), answers),
exception=TypeError)
# add optional split at the end
if self.add_newline:
for i in [
idx for idx, a in enumerate(answers)
if not a.endswith('\n')
]:
answers[i] += '\n'
return answers
def test_ensure_ascii_string(self):
"""Tests for ensure_ascii_string function."""
unicode_txt = 'this -> ¢ <- is unicode'
test_cases = [
('', ''),
('foo', 'foo'),
([1, 2, 3], "[1, 2, 3]"),
(['1', '2', '3'], "['1', '2', '3']"),
({
'one': 1
}, "{'one': 1}"),
# in both Python 2 & 3, Unicode characters that are part of a non-string value get escaped
([unicode_txt], "['this -> \\xc2\\xa2 <- is unicode']"),
({
'foo': unicode_txt
}, "{'foo': 'this -> \\xc2\\xa2 <- is unicode'}"),
]
if is_py2():
test_cases.extend([
# Unicode characters from regular strings are stripped out in Python 2
(unicode_txt, 'this -> <- is unicode'),
# also test with unicode-type values (only exists in Python 2)
(unicode('foo'), 'foo'),
(unicode(unicode_txt,
encoding='utf-8'), 'this -> \\xa2 <- is unicode'),
])
else:
# in Python 3, Unicode characters are replaced by backslashed escape sequences in string values
expected_unicode_out = 'this -> \\xc2\\xa2 <- is unicode'
test_cases.extend([
(unicode_txt, expected_unicode_out),
# also test with bytestring-type values (only exists in Python 3)
(bytes('foo', encoding='utf-8'), 'foo'),
(bytes(unicode_txt, encoding='utf-8'), expected_unicode_out),
])
for inp, out in test_cases:
res = ensure_ascii_string(inp)
self.assertTrue(is_string(res))
self.assertEqual(res, out)
def znode_path(self, znode=None):
"""Create znode path and make sure is subtree of BASE_ZNODE"""
base_znode_string = self.BASE_ZNODE
if znode is None:
znode = base_znode_string
elif isinstance(znode, (tuple, list)):
znode = '/'.join(znode)
if is_string(znode):
if not znode.startswith(base_znode_string):
if not znode.startswith('/'):
znode = '%s/%s' % (self.BASE_ZNODE, znode)
else:
self.log.raiseException('path %s not subpath of %s ' %
(znode, base_znode_string))
else:
self.log.raiseException('Unsupported znode type %s (znode %s)' %
(znode, type(znode)))
self.log.debug("znode %s" % znode)
return znode
def __init__(self, nrange):
"""Initialisation
@param nrange: nrange in [@][start:][end] format. If it is not a string, it is converted to
string and that string should allow conversion to float.
"""
self.log = getLogger(self.__class__.__name__, fname=False)
if not is_string(nrange):
newnrange = str(nrange)
self.log.debug("nrange %s of type %s, converting to string (%s)",
str(nrange), type(nrange), newnrange)
try:
float(newnrange)
except ValueError:
self.log.raiseException(
"nrange %s (type %s) is not valid after conversion to string (newnrange %s)"
% (str(nrange), type(nrange), newnrange))
nrange = newnrange
self.range_fn = self.parse(nrange)
def _init_input(self):
"""Handle input, if any in a simple way"""
if self.input is not None: # allow empty string (whatever it may mean)
# in Python 3, stdin.write requires a bytestring
if is_py3() and is_string(self.input):
inp = bytes(self.input, encoding='utf-8')
else:
inp = self.input
try:
self._process.stdin.write(inp)
except Exception:
self.log.raiseException(
"_init_input: Failed write input %s to process" %
self.input)
if self.INIT_INPUT_CLOSE:
self._process.stdin.close()
self.log.debug("_init_input: process stdin closed")
else:
self.log.debug("_init_input: process stdin NOT closed")
def _make_shell_command(self):
"""Convert cmd into shell command"""
self.log.warning(
"using potentialy unsafe shell commands, use run.run or run.RunNoShell.run \
instead of run.run_simple or run.Run.run")
if self.cmd is None:
self.log.raiseException("_make_shell_command: no cmd set.")
if is_string(self.cmd):
self._shellcmd = self.cmd
elif isinstance(self.cmd, (
list,
tuple,
)):
self._shellcmd = " ".join(
[str(arg).replace(' ', '\ ') for arg in self.cmd])
else:
self.log.raiseException(
"Failed to convert cmd %s (type %s) into shell command" %
(self.cmd, type(self.cmd)))
def streamLog(self, levelno, data):
"""
Add (continuous) data to an existing message stream (eg a stream after a logging.info()
"""
if is_string(levelno):
levelno = getLevelInt(levelno)
def write_and_flush_stream(hdlr, data=None):
"""Write to stream and flush the handler"""
if (not hasattr(hdlr, 'stream')) or hdlr.stream is None:
# no stream or not initialised.
raise Exception(
"write_and_flush_stream failed. No active stream attribute."
)
if data is not None:
hdlr.stream.write(data)
hdlr.flush()
# only log when appropriate (see logging.Logger.log())
if self.isEnabledFor(levelno):
self._handleFunction(write_and_flush_stream, levelno, data=data)
def convert_to_datetime(timestamp=None):
"""
Convert a string or datetime.datime instance to a datetime.datetime with UTC tzinfo
If no timestamp is given return current time
if timestamp is a string we can convert following formats:
Parse a datestamp according to its length
* YYYYMMDD (8 chars)
* unix timestamp (10 chars or any int)
* YYYYMMDDHHMM (12 chars)
* LDAP_DATETIME_TIMEFORMAT
"""
if timestamp is None:
# utcnow is time tuple with valid utc time without tzinfo
# replace(tzinfo=utc) fixes the tzinfo
return datetime.datetime.utcnow().replace(tzinfo=utc)
if isinstance(timestamp, int):
timestamp = "%010d" % timestamp
if isinstance(timestamp, datetime.datetime):
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=utc)
elif is_string(timestamp):
if len(timestamp) == 10:
# Unix timestamp
timestamp = datetime.datetime.fromtimestamp(int(timestamp), utc)
else:
if len(timestamp) == 12:
date_format = "%Y%m%d%H%M"
elif len(timestamp
) == 15: # len(LDAP_DATETIME_FORMAT doesn't work here
date_format = LDAP_DATETIME_TIMEFORMAT
elif len(timestamp) == 8:
date_format = "%Y%m%d"
else:
raise Exception("invalid format provided %s" % timestamp)
timestamp = datetime.datetime.strptime(timestamp, date_format)
return timestamp.replace(tzinfo=utc)
def __init__(self, endings=None):
if is_string(endings):
endings = [endings]
elif endings is None:
endings = []
self.endings = tuple(map(str, endings))
def autocomplete(parser,
arg_completer=None,
opt_completer=None,
subcmd_completer=None,
subcommands=None):
"""Automatically detect if we are requested completing and if so generate
completion automatically from given parser.
'parser' is the options parser to use.
'arg_completer' is a callable object that gets invoked to produce a list of
completions for arguments completion (oftentimes files).
'opt_completer' is the default completer to the options that require a
value.
'subcmd_completer' is the default completer for the subcommand
arguments.
If 'subcommands' is specified, the script expects it to be a map of
command-name to an object of any kind. We are assuming that this object is
a map from command name to a pair of (options parser, completer) for the
command. If the value is not such a tuple, the method
'autocomplete(completer)' is invoked on the resulting object.
This will attempt to match the first non-option argument into a subcommand
name and if so will use the local parser in the corresponding map entry's
value. This is used to implement completion for subcommand syntax and will
not be needed in most cases.
"""
# If we are not requested for complete, simply return silently, let the code
# caller complete. This is the normal path of execution.
if OPTCOMPLETE_ENVIRONMENT not in os.environ:
return
# After this point we should never return, only sys.exit(1)
# Set default completers.
if arg_completer is None:
arg_completer = NoneCompleter()
if opt_completer is None:
opt_completer = FileCompleter()
if subcmd_completer is None:
# subcmd_completer = arg_completer
subcmd_completer = FileCompleter()
# By default, completion will be arguments completion, unless we find out
# later we're trying to complete for an option.
completer = arg_completer
#
# Completing...
#
# Fetching inputs... not sure if we're going to use these.
# zsh's bashcompinit does not pass COMP_WORDS, replace with
# COMP_LINE for now...
if 'COMP_WORDS' not in os.environ:
os.environ['COMP_WORDS'] = os.environ['COMP_LINE']
cwords = shlex.split(os.environ.get('COMP_WORDS', '').strip('() '))
cline = os.environ.get('COMP_LINE', '')
cpoint = int(os.environ.get('COMP_POINT', 0))
cword = int(os.environ.get('COMP_CWORD', 0))
# Extract word enclosed word.
prefix, suffix = extract_word(cline, cpoint)
# If requested, try subcommand syntax to find an options parser for that
# subcommand.
if subcommands:
assert isinstance(subcommands, dict)
value = guess_first_nonoption(parser, subcommands)
if value:
if isinstance(value, (list, tuple)):
parser = value[0]
if len(value) > 1 and value[1]:
# override completer for command if it is present.
completer = value[1]
else:
completer = subcmd_completer
autocomplete(parser, completer)
elif hasattr(value, 'autocomplete'):
# Call completion method on object. This should call
# autocomplete() recursively with appropriate arguments.
value.autocomplete(subcmd_completer)
else:
# no completions for that command object
pass
sys.exit(1)
else: # suggest subcommands
completer = ListCompleter(subcommands.keys())
# Look at previous word, if it is an option and it requires an argument,
# check for a local completer. If there is no completer, what follows
# directly cannot be another option, so mark to not add those to
# completions.
optarg = False
try:
# Look for previous word, which will be containing word if the option
# has an equals sign in it.
prev = None
if cword < len(cwords):
mo = re.search('(--.*?)=(.*)', cwords[cword])
if mo:
prev, prefix = mo.groups()
if not prev:
prev = cwords[cword - 1]
if prev and prev.startswith('-'):
option = parser.get_option(prev)
if option:
if option.nargs > 0:
optarg = True
if hasattr(option, 'completer'):
completer = option.completer
elif option.choices:
completer = ListCompleter(option.choices)
elif option.type in ('string', ):
completer = opt_completer
else:
completer = NoneCompleter()
# Warn user at least, it could help him figure out the problem.
elif hasattr(option, 'completer'):
msg = "Error: optparse option with a completer does not take arguments: %s" % (
option)
raise SystemExit(msg)
except KeyError:
pass
completions = []
# Options completion.
if not optarg and (not prefix or prefix.startswith('-')):
completions += parser._short_opt.keys()
completions += parser._long_opt.keys()
# Note: this will get filtered properly below.
completer_kwargs = {
'pwd': os.getcwd(),
'cline': cline,
'cpoint': cpoint,
'prefix': prefix,
'suffix': suffix,
}
# File completion.
if completer and (not prefix or not prefix.startswith('-')):
# Call appropriate completer depending on type.
if isinstance(completer, (list, tuple)) or is_string(completer):
completer = FileCompleter(completer)
elif not isinstance(completer, (types.FunctionType, types.LambdaType,
types.ClassType, types.ObjectType)):
# TODO: what to do here?
pass
completions = completer(**completer_kwargs)
if is_string(completions):
# is a bash command, just run it
if SHELL in (BASH, ): # TODO: zsh
print(completions)
else:
raise Exception("Commands are unsupported by this shell %s" %
SHELL)
else:
# Filter using prefix.
if prefix:
completions = sorted(
filter(lambda x: x.startswith(prefix), completions))
completions = ' '.join(map(str, completions))
# Save results
if SHELL == "bash":
print('COMPREPLY=(' + completions + ')')
else:
print(completions)
# Print debug output (if needed). You can keep a shell with 'tail -f' to
# the log file to monitor what is happening.
if debugfn:
txt = "\n".join([
'---------------------------------------------------------',
'CWORDS %s' % cwords,
'CLINE %s' % cline,
'CPOINT %s' % cpoint,
'CWORD %s' % cword,
'',
'Short options',
pformat(parser._short_opt),
'',
'Long options',
pformat(parser._long_opt),
'Prefix %s' % prefix,
'Suffix %s' % suffix,
'completer_kwargs%s' % str(completer_kwargs),
#'completer_completions %s' % completer_completions,
'completions %s' % completions,
])
if isinstance(debugfn, logging.Logger):
debugfn.debug(txt)
else:
f = open(debugfn, 'a')
f.write(txt)
f.close()
# Exit with error code (we do not let the caller continue on purpose, this
# is a run for completions only.)
sys.exit(1)