def test_env_add_folded_pre(self):
"""
TEST: add to a preset environment variable from a folded [env]
entry
EXP: the value gets set to the payload with the whitespace squeezed out
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
pre_val = "one:two:three"
add = "four:\n five:\n six"
exp = ":".join([pre_val, re.sub("\n\s*", "", add)])
# make sure the target env variable has the expected value
with util.tmpenv(evname, pre_val):
# create a config object with an 'env' section and a folded '+'
# option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, '+' + add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the variable was set to the expected value
self.expected(exp, os.environ[evname])
def test_env_set_folded_none(self):
"""
TEST: set undefined environment variable from a folded [env] entry
unconditionally
EXP: the value gets set
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
newval = "one:\n two:\n three"
exp = re.sub("\n\s*", "", newval)
# make sure the target env variable is not defined
with util.tmpenv(evname, None):
# create a config object with an 'env' section and a non-'+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, newval)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the variable was set to the expected value
self.expected(exp, os.environ[evname])
def test_env_add_pre(self):
"""
TEST: add to a predefined environment variable from [env] entry
EXP: payload is appended to the old value
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
pre_val = "one:two:three"
add = "four:five:six"
exp = ":".join([pre_val, add])
# make sure the target env variable is set to a known value
with util.tmpenv(evname, pre_val):
# create a config object with an 'env' section and a '+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, "+" + add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the target env variable now contains both old and
# added values
self.expected(exp, os.environ[evname])
def test_env_set_folded_none(self):
"""
TEST: set undefined environment variable from a folded [env] entry
unconditionally
EXP: the value gets set
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
newval = "one:\n two:\n three"
exp = re.sub("\n\s*", "", newval)
# make sure the target env variable is not defined
with util.tmpenv(evname, None):
# create a config object with an 'env' section and a non-'+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, newval)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the variable was set to the expected value
self.expected(exp, os.environ[evname])
def test_env_set_pre(self):
"""
TEST: set predefined environment variable from [env] entry
unconditionally
EXP: the old value gets overwritten
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
pre_val = "one:two:three"
add = "four:five:six"
exp = add
# make sure the target env variable is set to a known value
with util.tmpenv(evname, pre_val):
# create a config object with an 'env' section and a non-'+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the target env variable now contains the new value
# and the old value is gone
self.expected(exp, os.environ[evname])
self.assertTrue(pre_val not in os.environ[evname],
"The old value should be gone but still seems " +
"to be hanging around")
def test_env_add_folded_pre(self):
"""
TEST: add to a preset environment variable from a folded [env]
entry
EXP: the value gets set to the payload with the whitespace squeezed out
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
pre_val = "one:two:three"
add = "four:\n five:\n six"
exp = ":".join([pre_val, re.sub("\n\s*", "", add)])
# make sure the target env variable has the expected value
with util.tmpenv(evname, pre_val):
# create a config object with an 'env' section and a folded '+'
# option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, '+' + add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the variable was set to the expected value
self.expected(exp, os.environ[evname])
def test_env_add_pre(self):
"""
TEST: add to a predefined environment variable from [env] entry
EXP: payload is appended to the old value
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
pre_val = "one:two:three"
add = "four:five:six"
exp = ":".join([pre_val, add])
# make sure the target env variable is set to a known value
with util.tmpenv(evname, pre_val):
# create a config object with an 'env' section and a '+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, "+" + add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the target env variable now contains both old and
# added values
self.expected(exp, os.environ[evname])
def test_env_add_folded_none(self):
"""
TEST: add to an undefined environment variable from a folded [env]
entry
EXP: the value gets set to the payload with the whitespace squeezed out
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
add = "four:\n five:\n six"
exp = re.sub("\n\s*", "", add)
# make sure the target env variable is not defined
with util.tmpenv(evname, None):
# create a config object with an 'env' section and a '+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, '+' + add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the variable was set to the expected value
self.expected(exp, os.environ[evname])
def test_env_set_pre(self):
"""
TEST: set predefined environment variable from [env] entry
unconditionally
EXP: the old value gets overwritten
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
pre_val = "one:two:three"
add = "four:five:six"
exp = add
# make sure the target env variable is set to a known value
with util.tmpenv(evname, pre_val):
# create a config object with an 'env' section and a non-'+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the target env variable now contains the new value
# and the old value is gone
self.expected(exp, os.environ[evname])
self.assertTrue(
pre_val not in os.environ[evname],
"The old value should be gone but still seems " +
"to be hanging around")
def test_expand_filled(self):
"""
Test expand on
'before $VAR after',
'before/${VAR}/after',
'before.${VAR:-default-value}.after'
Note that the default value is allowed but not used in the expansion.
Python does not natively support that aspect of shell variable
expansion.
"""
self.dbgfunc()
vname = "EXPAND_FILLED"
with util.tmpenv(vname, "SOMETHING"):
bare_str = "before $%s after" % vname
exp = "before SOMETHING after"
actual = util.expand(bare_str)
self.expected(exp, actual)
braced_str = "before/${%s}/after" % vname
exp = "before/SOMETHING/after"
actual = util.expand(braced_str)
self.expected(exp, actual)
def_str = "before.${%s:-default-value}.after" % vname
exp = "before.SOMETHING.after"
actual = util.expand(def_str)
self.expected(exp, actual)
def test_env_add_folded_none(self):
"""
TEST: add to an undefined environment variable from a folded [env]
entry
EXP: the value gets set to the payload with the whitespace squeezed out
"""
self.dbgfunc()
sname = 'env'
evname = 'UTIL_TEST'
add = "four:\n five:\n six"
exp = re.sub("\n\s*", "", add)
# make sure the target env variable is not defined
with util.tmpenv(evname, None):
# create a config object with an 'env' section and a '+' option
cfg = CrawlConfig.CrawlConfig()
cfg.add_section(sname)
cfg.set(sname, evname, '+' + add)
# pass the config object to util.env_update()
util.env_update(cfg)
# verify that the variable was set to the expected value
self.expected(exp, os.environ[evname])
def test_expand_filled(self):
"""
Test expand on
'before $VAR after',
'before/${VAR}/after',
'before.${VAR:-default-value}.after'
Note that the default value is allowed but not used in the expansion.
Python does not natively support that aspect of shell variable
expansion.
"""
self.dbgfunc()
vname = "EXPAND_FILLED"
with util.tmpenv(vname, "SOMETHING"):
bare_str = "before $%s after" % vname
exp = "before SOMETHING after"
actual = util.expand(bare_str)
self.expected(exp, actual)
braced_str = "before/${%s}/after" % vname
exp = "before/SOMETHING/after"
actual = util.expand(braced_str)
self.expected(exp, actual)
def_str = "before.${%s:-default-value}.after" % vname
exp = "before.SOMETHING.after"
actual = util.expand(def_str)
self.expected(exp, actual)
def test_maybe_update_hsi_no(muh_prep, tmpdir):
"""
If hsi matches mostly except for the 'exec' statement, it should not get
updated
"""
pytest.dbgfunc()
rf = test_maybe_update_hsi_no
path = ":".join([rf.bin.strpath, rf.hsihome])
with U.tmpenv('PATH', path):
hpss.maybe_update_hsi()
assert abs(rf.then - rf.file.mtime()) < 1.0
c = rf.file.read()
assert 'exec ${EXECUTABLE}' in c
def test_maybe_update_hsi_yes(muh_prep, tmpdir):
"""
If the local hsi wrapper is sufficiently different, maybe_update should do
the update
"""
pytest.dbgfunc()
rf = test_maybe_update_hsi_yes
path = ":".join([rf.bin.strpath, rf.hsihome])
with U.tmpenv('PATH', path):
hpss.maybe_update_hsi()
assert abs(time.time() - rf.file.mtime()) < 1.0
c = rf.file.read()
assert "not changed" not in c
assert "exec ${EXECUTABLE}" in c
def test_maybe_update_hsi_cant(muh_prep, tmpdir):
"""
If we don't have write permission on the target, then even if we should
update, we can't. In this case, should log a message.
"""
pytest.dbgfunc()
lp = tmpdir.join('crawl.test.log')
rf = test_maybe_update_hsi_cant
path = ":".join([rf.bin.strpath, rf.hsihome])
with U.tmpenv('PATH', path):
CrawlConfig.log(logpath=lp.strpath, close=True)
hpss.maybe_update_hsi()
c = rf.file.read()
assert 'not changed' in c
assert os.path.exists(lp.strpath)
c = lp.read()
assert MSG.hsi_wrap_ood in c
CrawlConfig.log(close=True)
def test_which_all(hits, xable, tmpdir):
"""
Test U.which_all() with no matches in $PATH
"""
pytest.dbgfunc()
hitname = "runnable"
for n in range(hits):
tmpdir.make_numbered_dir(prefix="bin_", rootdir=tmpdir)
path = ":".join([p.strpath for p in tmpdir.listdir()])
exp = []
for n, b in enumerate(tmpdir.listdir()):
f = b.join(hitname).ensure()
if n < xable:
f.chmod(0755)
exp.append(f.strpath)
with U.tmpenv('PATH', path):
result = U.which_all(hitname)
assert len(result) == xable
assert sorted(exp) == sorted(result)
def test_which_all(hits, xable, tmpdir):
"""
Test U.which_all() with no matches in $PATH
"""
pytest.dbgfunc()
hitname = "runnable"
for n in range(hits):
tmpdir.make_numbered_dir(prefix="bin_", rootdir=tmpdir)
path = ":".join([p.strpath for p in tmpdir.listdir()])
exp = []
for n, b in enumerate(tmpdir.listdir()):
f = b.join(hitname).ensure()
if n < xable:
f.chmod(0755)
exp.append(f.strpath)
with U.tmpenv('PATH', path):
result = U.which_all(hitname)
assert len(result) == xable
assert sorted(exp) == sorted(result)
def test_alert_email_defcfg(self):
"""
Generate an e-mail alert using the default config and verify that it
was sent (this is where we use 'monkey patching').
"""
self.dbgfunc()
fakesmtp.inbox = []
CrawlConfig.add_config(close=True)
# with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'):
with U.tmpenv('CRAWL_CONF', None):
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='cv', msg=payload)
m = fakesmtp.inbox[0]
self.expected(', '.join(m.to_address), targets)
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
def test_expand_empty(self):
"""
Test expanding an empty variable with simple, braced, and default
syntaxes
"""
self.dbgfunc()
vname = "EXPAND_EMPTY"
with util.tmpenv(vname, ""):
bare_str = "before $%s after" % vname
exp = "before after"
actual = util.expand(bare_str)
self.expected(exp, actual)
braced_str = "before/${%s}/after" % vname
exp = "before//after"
actual = util.expand(braced_str)
self.expected(exp, actual)
def_str = "before.${%s:-default-value}.after" % vname
exp = "before..after"
actual = util.expand(def_str)
self.expected(exp, actual)
def test_expand_empty(self):
"""
Test expanding an empty variable with simple, braced, and default
syntaxes
"""
self.dbgfunc()
vname = "EXPAND_EMPTY"
with util.tmpenv(vname, ""):
bare_str = "before $%s after" % vname
exp = "before after"
actual = util.expand(bare_str)
self.expected(exp, actual)
braced_str = "before/${%s}/after" % vname
exp = "before//after"
actual = util.expand(braced_str)
self.expected(exp, actual)
def_str = "before.${%s:-default-value}.after" % vname
exp = "before..after"
actual = util.expand(def_str)
self.expected(exp, actual)
def test_alert_email_defcfg(self):
"""
Generate an e-mail alert using the default config and verify that it
was sent (this is where we use 'monkey patching').
"""
self.dbgfunc()
fakesmtp.inbox = []
CrawlConfig.add_config(close=True)
# with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'):
with U.tmpenv('CRAWL_CONF', None):
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='cv', msg=payload)
m = fakesmtp.inbox[0]
self.expected(', '.join(m.to_address), targets)
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
