def test_whitelisted_characters_overlap_blacklisted_characters():
good_chars = u'te02тест49st'
bad_chars = u'ts94тсет'
with pytest.raises(InvalidArgument) as exc:
characters(min_codepoint=ord('0'), max_codepoint=ord('9'),
whitelist_characters=good_chars,
blacklist_characters=bad_chars).example()
assert repr(good_chars) in text_type(exc)
assert repr(bad_chars) in text_type(exc)
def test_arbitrary_blacklist(data):
blacklist = data.draw(st.text(st.characters(max_codepoint=1000), min_size=1))
ords = list(map(ord, blacklist))
c = data.draw(
st.characters(
blacklist_characters=blacklist,
min_codepoint=max(0, min(ords) - 1),
max_codepoint=max(0, max(ords) + 1),
)
)
assert c not in blacklist
def random_prefix(draw):
#limited to unicode letters, see
#https://en.wikipedia.org/wiki/Unicode_character_property#General_Category
categories = ['Ll', 'Lt', 'Lm', 'Lo']
characters = st.lists(st.characters(whitelist_categories=categories), min_size = 1)
prefix = st.text(alphabet = draw(characters), min_size = 1)
return draw(prefix)
def test_characters_of_specific_groups():
st = characters(whitelist_categories=("Lu", "Nd"))
find_any(st, lambda c: unicodedata.category(c) == "Lu")
find_any(st, lambda c: unicodedata.category(c) == "Nd")
assert_no_examples(st, lambda c: unicodedata.category(c) not in ("Lu", "Nd"))
def test_exclude_characters_of_specific_groups():
st = characters(blacklist_categories=("Lu", "Nd"))
find_any(st, lambda c: unicodedata.category(c) != "Lu")
find_any(st, lambda c: unicodedata.category(c) != "Nd")
assert_no_examples(st, lambda c: unicodedata.category(c) in ("Lu", "Nd"))
def test_exclude_characters_of_specific_groups():
st = characters(blacklist_categories=('Lu', 'Nd'))
find(st, lambda c: unicodedata.category(c) != 'Lu')
find(st, lambda c: unicodedata.category(c) != 'Nd')
assert_no_examples(st, lambda c: unicodedata.category(c) in ('Lu', 'Nd'))
def test_find_something_rare():
st = characters(whitelist_categories=['Zs'], min_codepoint=12288)
find(st, lambda c: unicodedata.category(c) == 'Zs')
with pytest.raises(NoSuchExample):
find(st, lambda c: unicodedata.category(c) != 'Zs')
def _for_text(field):
# We can infer a vastly more precise strategy by considering the
# validators as well as the field type. This is a minimal proof of
# concept, but we intend to leverage the idea much more heavily soon.
# See https://github.com/HypothesisWorks/hypothesis-python/issues/1116
regexes = [
re.compile(v.regex, v.flags) if isinstance(v.regex, str) else v.regex
for v in field.validators
if isinstance(v, django.core.validators.RegexValidator) and not v.inverse_match
]
if regexes:
# This strategy generates according to one of the regexes, and
# filters using the others. It can therefore learn to generate
# from the most restrictive and filter with permissive patterns.
# Not maximally efficient, but it makes pathological cases rarer.
# If you want a challenge: extend https://qntm.org/greenery to
# compute intersections of the full Python regex language.
return st.one_of(*[st.from_regex(r) for r in regexes])
# If there are no (usable) regexes, we use a standard text strategy.
min_size = 1
if getattr(field, "blank", False) or not getattr(field, "required", True):
min_size = 0
strategy = st.text(
alphabet=st.characters(
blacklist_characters=u"\x00", blacklist_categories=("Cs",)
),
min_size=min_size,
max_size=field.max_length,
)
if getattr(field, "required", True):
strategy = strategy.filter(lambda s: s.strip())
return strategy
def fstrings(draw):
"""
Generate a valid f-string.
See https://www.python.org/dev/peps/pep-0498/#specification
:param draw: Let hypothsis draw from other strategies.
:return: A valid f-string.
"""
character_strategy = st.characters(
blacklist_characters='\r\n\'\\s{}',
min_codepoint=1,
max_codepoint=1000,
)
is_raw = draw(st.booleans())
integer_strategy = st.integers(min_value=0, max_value=3)
expression_count = draw(integer_strategy)
content = []
for _ in range(expression_count):
expression = draw(expressions())
conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
has_specifier = draw(st.booleans())
specifier = ':' + draw(format_specifiers()) if has_specifier else ''
content.append('{{{}{}}}'.format(expression, conversion, specifier))
content.append(draw(st.text(character_strategy)))
content = ''.join(content)
return "f{}'{}'".format('r' if is_raw else '', content)
def test_exclude_characters_of_specific_groups():
st = characters(blacklist_categories=('Lu', 'Nd'))
find(st, lambda c: unicodedata.category(c) != 'Lu')
find(st, lambda c: unicodedata.category(c) != 'Nd')
with pytest.raises(NoSuchExample):
find(st, lambda c: unicodedata.category(c) in ('Lu', 'Nd'))
def test_characters_of_specific_groups():
st = characters(whitelist_categories=('Lu', 'Nd'))
find(st, lambda c: unicodedata.category(c) == 'Lu')
find(st, lambda c: unicodedata.category(c) == 'Nd')
with pytest.raises(NoSuchExample):
find(st, lambda c: unicodedata.category(c) not in ('Lu', 'Nd'))
def test_characters_of_specific_groups():
st = characters(whitelist_categories=("Lu", "Nd"))
find(st, lambda c: unicodedata.category(c) == "Lu")
find(st, lambda c: unicodedata.category(c) == "Nd")
with pytest.raises(NoSuchExample):
find(st, lambda c: unicodedata.category(c) not in ("Lu", "Nd"))
def test_whitelist_characters_disjoint_blacklist_characters():
good_chars = u'123abc'
bad_chars = u'456def'
st = characters(min_codepoint=ord('0'), max_codepoint=ord('9'),
blacklist_characters=bad_chars,
whitelist_characters=good_chars)
assert_no_examples(st, lambda c: c in bad_chars)
def test_blacklisted_characters():
bad_chars = u"te02тест49st"
st = characters(min_codepoint=ord("0"), max_codepoint=ord("9"), blacklist_characters=bad_chars)
assert "1" == find(st, lambda c: True)
with pytest.raises(NoSuchExample):
find(st, lambda c: c in bad_chars)
def test_blacklisted_characters():
bad_chars = u'te02тест49st'
st = characters(min_codepoint=ord('0'), max_codepoint=ord('9'),
blacklist_characters=bad_chars)
assert '1' == find(st, lambda c: True)
assert_no_examples(st, lambda c: c in bad_chars)
def textlists():
"""
Strategy for generating lists storable with L{axiom.attributes.textlist}.
"""
return st.lists(st.text(
alphabet=st.characters(
blacklist_categories={'Cs'},
blacklist_characters={u'\x00', u'\x02', u'\x1f'})))
def test_characters_of_specific_groups():
st = characters(whitelist_categories=('Lu', 'Nd'))
find(st, lambda c: unicodedata.category(c) == 'Lu')
find(st, lambda c: unicodedata.category(c) == 'Nd')
assert_no_examples(
st, lambda c: unicodedata.category(c) not in ('Lu', 'Nd'))
def test_whitelisted_characters_override():
good_characters = u'teтестst'
st = characters(min_codepoint=ord('0'), max_codepoint=ord('9'),
whitelist_characters=good_characters)
find_any(st, lambda c: c in good_characters)
find_any(st, lambda c: c in '0123456789')
assert_no_examples(st, lambda c: c not in good_characters + '0123456789')
def test_blacklisted_characters():
bad_chars = u'te02тест49st'
st = characters(min_codepoint=ord('0'), max_codepoint=ord('9'),
blacklist_characters=bad_chars)
assert '1' == find(st, lambda c: True)
with pytest.raises(NoSuchExample):
find(st, lambda c: c in bad_chars)
def test_blacklisted_characters():
bad_chars = u"te02тест49st"
st = characters(
min_codepoint=ord("0"), max_codepoint=ord("9"), blacklist_characters=bad_chars
)
assert "1" == minimal(st, lambda c: True)
assert_no_examples(st, lambda c: c in bad_chars)
def strategy(self):
"""Returns resulting strategy that generates configured char set."""
max_codepoint = None if self._unicode else 127
if self._negate:
black_chars = self._blacklist_chars - self._whitelist_chars
return st.characters(
blacklist_categories=self._categories | {"Cc", "Cs"},
blacklist_characters=self._whitelist_chars,
whitelist_characters=black_chars,
max_codepoint=max_codepoint,
)
white_chars = self._whitelist_chars - self._blacklist_chars
return st.characters(
whitelist_categories=self._categories,
blacklist_characters=self._blacklist_chars,
whitelist_characters=white_chars,
max_codepoint=max_codepoint,
)
def axiomText(*a, **kw):
"""
Strategy for generating Axiom-compatible text values.
"""
return st.text(
alphabet=st.characters(
blacklist_categories={'Cs'},
blacklist_characters={u'\x00'}),
*a, **kw)
def _get_strategy_for_field(f):
# type: (Type[dm.Field]) -> st.SearchStrategy[Any]
if f.choices:
choices = [] # type: list
for value, name_or_optgroup in f.choices:
if isinstance(name_or_optgroup, (list, tuple)):
choices.extend(key for key, _ in name_or_optgroup)
else:
choices.append(value)
if isinstance(f, (dm.CharField, dm.TextField)) and f.blank:
choices.insert(0, u'')
strategy = st.sampled_from(choices)
elif type(f) == dm.SlugField:
strategy = st.text(alphabet=string.ascii_letters + string.digits,
min_size=(0 if f.blank else 1),
max_size=f.max_length)
elif type(f) == dm.GenericIPAddressField:
lookup = {'both': ip4_addr_strings() | ip6_addr_strings(),
'ipv4': ip4_addr_strings(), 'ipv6': ip6_addr_strings()}
strategy = lookup[f.protocol.lower()]
elif type(f) in (dm.TextField, dm.CharField):
strategy = st.text(
alphabet=st.characters(blacklist_characters=u'\x00',
blacklist_categories=('Cs',)),
min_size=(0 if f.blank else 1),
max_size=f.max_length,
)
# We can infer a vastly more precise strategy by considering the
# validators as well as the field type. This is a minimal proof of
# concept, but we intend to leverage the idea much more heavily soon.
# See https://github.com/HypothesisWorks/hypothesis-python/issues/1116
re_validators = [
v for v in f.validators
if isinstance(v, validators.RegexValidator) and not v.inverse_match
]
if re_validators:
regexes = [re.compile(v.regex, v.flags) if isinstance(v.regex, str)
else v.regex for v in re_validators]
# This strategy generates according to one of the regexes, and
# filters using the others. It can therefore learn to generate
# from the most restrictive and filter with permissive patterns.
# Not maximally efficient, but it makes pathological cases rarer.
# If you want a challenge: extend https://qntm.org/greenery to
# compute intersections of the full Python regex language.
strategy = st.one_of(*[st.from_regex(r) for r in regexes])
elif type(f) == dm.DecimalField:
bound = Decimal(10 ** f.max_digits - 1) / (10 ** f.decimal_places)
strategy = st.decimals(min_value=-bound, max_value=bound,
places=f.decimal_places)
else:
strategy = field_mappings().get(type(f), st.nothing())
if f.validators:
strategy = strategy.filter(validator_to_filter(f))
if f.null:
strategy = st.one_of(st.none(), strategy)
return strategy
def schema2strategies(schema):
types = {
'Character' : st.characters(),
'Flag' : st.booleans(),
'Integer' : st.integers(),
'Float' : st.floats(),
'String' : st.text(max_size=10)
}
strategy = types[schema['Type']]
return schema['ID'], strategy
def perturbed_by_character(draw, string_strategy):
"""
A strategy that constructs a string using the supplied ``string_strategy``,
and then perturbs it by a single character.
"""
serialized = draw(string_strategy)
operation = draw(strategies.sampled_from((insert, replace, delete)))
index = draw(strategies.floats(min_value=0, max_value=1))
character = draw(strategies.characters())
return operation(serialized, int(len(serialized) * index), character)
def test_whitelist_characters_disjoint_blacklist_characters():
good_chars = u"123abc"
bad_chars = u"456def"
st = characters(
min_codepoint=ord("0"),
max_codepoint=ord("9"),
blacklist_characters=bad_chars,
whitelist_characters=good_chars,
)
assert_no_examples(st, lambda c: c in bad_chars)
def test_whitelisted_characters_override():
good_characters = u"teтестst"
st = characters(
min_codepoint=ord("0"),
max_codepoint=ord("9"),
whitelist_characters=good_characters,
)
find_any(st, lambda c: c in good_characters)
find_any(st, lambda c: c in "0123456789")
assert_no_examples(st, lambda c: c not in good_characters + "0123456789")
def nice_strings(bad_chars):
"""
A Hypothesis strategy to generate reasonable name strings
"""
def sane_first_character(s):
c = s[0]
return c not in ['/']
return text(alphabet=characters(blacklist_characters=bad_chars,
whitelist_categories=['Ll'],
min_codepoint=ord('0'),
max_codepoint=ord('z')),
min_size=1).filter(sane_first_character)
def aws_keypair_name():
# So far as I can tell, based on ``aws ec2 create-key-pair help``
return strategies.lists(
strategies.characters(
min_codepoint=0,
max_codepoint=255,
),
min_size=1,
average_size=8,
max_size=255,
).map(
u"".join
)
def urls():
"""
Strategy for generating ``twisted.python.url.URL``\s.
"""
return s.builds(
URL,
scheme=s.just(u'https'),
host=dns_names(),
path=s.lists(s.text(
max_size=64,
alphabet=s.characters(blacklist_characters=u'/?#',
blacklist_categories=('Cs',))
), min_size=1, max_size=10))
import hypothesis.extra.numpy as npst
try:
import hypothesis.extra.pytz as tzst
except ImportError:
tzst = None
try:
import zoneinfo
except ImportError:
zoneinfo = None
import numpy as np
import pyarrow as pa
# TODO(kszucs): alphanum_text, surrogate_text
custom_text = st.text(
alphabet=st.characters(min_codepoint=0x41, max_codepoint=0x7E))
null_type = st.just(pa.null())
bool_type = st.just(pa.bool_())
binary_type = st.just(pa.binary())
string_type = st.just(pa.string())
large_binary_type = st.just(pa.large_binary())
large_string_type = st.just(pa.large_string())
fixed_size_binary_type = st.builds(pa.binary,
st.integers(min_value=0, max_value=16))
binary_like_types = st.one_of(binary_type, string_type, large_binary_type,
large_string_type, fixed_size_binary_type)
signed_integer_types = st.sampled_from(
[pa.int8(), pa.int16(), pa.int32(),
#-------------------------------------------------------------------------------
# values
# 55203 is just before "high surrogates", and avoids this exception
# UnicodeDecodeError: 'utf-32-le' codec can't decode bytes in position 0-3: code point in surrogate code point range(0xd800, 0xe000)
ST_CODEPOINT_LIMIT = dict(min_codepoint=1, max_codepoint=55203)
ST_TYPES_COMMON: tp.Tuple[tp.Callable[..., st.SearchStrategy], ...] = (
st.integers,
# st.decimals,
st.fractions,
st.dates,
st.datetimes,
partial(st.characters, **ST_CODEPOINT_LIMIT),
partial(st.text, st.characters(**ST_CODEPOINT_LIMIT)) # type: ignore
)
ST_TYPES_FLOAT_NAN: tp.Tuple[st.SearchStrategy, ...] = (
st.floats,
st.complex_numbers,
)
filter_nan = lambda x: not np.isnan(x)
ST_TYPES_FLOAT_NO_NAN: tp.Tuple[tp.Callable[[], st.SearchStrategy], ...] = (
lambda: st.floats().filter(filter_nan),
lambda: st.complex_numbers().filter(filter_nan))
ST_TYPES_UNARY_BINARY = (st.booleans, st.none)
def test_exclude_characters_of_major_categories():
st = characters(blacklist_categories=("L", "N"))
find_any(st, lambda c: not unicodedata.category(c).startswith("L"))
find_any(st, lambda c: not unicodedata.category(c).startswith("N"))
assert_no_examples(st, lambda c: unicodedata.category(c)[0] in ("L", "N"))
import unittest
import hypothesis
import hypothesis.strategies as hst
import numpy as np
import kastore as kas
import kastore.store as store
# Set the deadline to None to avoid weird behaviour on CI.
hypothesis.settings.register_profile("kastore_defaults", deadline=None)
hypothesis.settings.load_profile("kastore_defaults")
# Exclude any 'other' unicode categories:
# http://www.unicode.org/reports/tr44/#General_Category_Values
key_alphabet = hst.characters(blacklist_categories=("C", ))
class TestFileSignature(unittest.TestCase):
"""
Checks the file signature is what we think it should be.
"""
def test_form(self):
self.assertEqual(len(store.MAGIC), 8)
self.assertEqual(b"\211KAS\r\n\032\n", store.MAGIC)
class FormatMixin:
"""
Tests for the file format.
"""
from hypothesis import strategies as st
py_atom = lambda: st.one_of(
st.none(),
st.booleans(),
st.integers(),
st.floats(),
st.complex_numbers(),
st.characters(),
st.binary(),
)
def py_value():
atom = py_atom()
return st.one_of(
atom,
st.lists(atom),
st.tuples(atom),
st.sets(atom),
st.dictionaries(atom, atom),
)
import pytest
from regexp_builder import grammar_to_regexp
from hypothesis import given
import hypothesis.strategies as some
@given(some.characters(whitelist_categories='L'))
def test_single_character_becomes_single_character(c):
test_input = f'0: "{c}"'
expected = f"^{c}$"
assert expected == grammar_to_regexp(test_input)
def test_a_single_rule_is_just_followed():
test_input = '''0: 1
1: "c"'''
expected = "^c$"
assert expected == grammar_to_regexp(test_input)
def test_two_rules_are_correctly_expanded():
test_input = '''0: 1 2
1: "a"
2: "f"'''
expected = "^af$"
assert expected == grammar_to_regexp(test_input)
with pytest.raises(ValueError, match="IDN"):
jid.JID.create("INVALID", "example\u200B.org", "foo/bar")
with pytest.raises(ValueError, match="UsernameCaseMapped"):
jid.JID.create("INVAL\u200BID", "example.org", "foo/bar")
with pytest.raises(ValueError, match="OpaqueString"):
jid.JID.create("INVALID", "example.org", "\u200B")
def test_dunders():
j1 = jid.JID.parse("porthos@銃士.lit")
with pytest.raises(AttributeError):
j1.local = "d'artagnan"
j2 = jid.JID.create("porthos", "xn--zqs335k.lit")
assert j1 == j2
assert hash(j1) == hash(j2)
_valid_xep_0106_ish = {
"whitelist_categories": ("Ll", "Lu", "Lo", "Nd", "Lm", "Mn", "Mc"),
"whitelist_characters": jid._XEP_0106_ESCAPE_SEQ,
}
@given(localpart=strat.characters(**_valid_xep_0106_ish))
def test_xep_0106(localpart):
if (localpart[0] == " " or localpart[-1]
== " "): # we test this one separately in test_create
return
assert jid._unescape_localpart(
jid._escape_localpart(localpart)) == localpart
expected_value)
@pytest.mark.parametrize(
"file_data, shape",
[
("NOPROP\n 1 5 3 7 2 6 4 8 /\n", (2, 2, 2)),
],
)
def test_read_values_raises_on_missing(file_data, shape):
with patch("builtins.open", mock_open(read_data=file_data)) as mock_file:
with pytest.raises(xtgeo.KeywordNotFoundError):
read_grdecl_3d_property(mock_file, "PROP", shape, int)
keywords = st.text(alphabet=st.characters(whitelist_categories=("Nd", "Lu")),
min_size=1)
grid_properties = arrays(elements=st.floats(),
dtype="float",
shape=st.tuples(indecies, indecies, indecies))
@given(keywords, grid_properties)
def test_read_write_grid_property_is_identity(keyword, grid_property):
values = [str(v) for v in grid_property.flatten(order="F")]
file_data = f"{keyword}\n {' '.join(values)} /"
with patch("builtins.open", mock_open(read_data=file_data)) as mock_file:
assert_allclose(
read_grdecl_3d_property(mock_file, keyword, grid_property.shape),
grid_property,
)
import numpy as np
import joblib
from hypothesis import given, note
from hypothesis import settings, strategies as st
from scilk.corpora import genia
from scilk.util import intervals
from scilk.collections import _collections
import scilk
MAX_TESTS = 1000
# strategies
texts = st.text(st.characters(min_codepoint=32, max_codepoint=255), 0, 500, 1000)
def loader_caller(collection: _collections.Collection, data=None):
def caller(value: str):
return collection.translate(value)
return caller
def loader_translate(collection: _collections.Collection, data: dict):
mapping = joblib.load(data['mapping'])
def translate(value: str):
return mapping.get(value)
def test_whitelisted_characters_alone():
with pytest.raises(InvalidArgument):
characters(whitelist_characters="te02тест49st").example()
def test_find_one():
char = minimal(characters(min_codepoint=48, max_codepoint=48),
lambda _: True)
assert char == "0"
filecharacters = (
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
"[]^_`;=@{}~ !#$%&'()+,-"
)
files = (
st.text(filecharacters, min_size=1)
.map(lambda x: x.strip())
.filter(bool)
.map(lambda s: s.encode("ascii"))
)
safetext = st.text(
st.characters(
min_codepoint=1, max_codepoint=127, blacklist_categories=("Cc", "Cs")
),
min_size=1,
).map(lambda s: s.encode("utf-8"))
extensions = st.sampled_from(("shelve", "mq", "blackbox"))
@contextmanager
def acceptableerrors(*args):
"""Sometimes we know an operation we're about to perform might fail, and
we're OK with some of the failures. In those cases this may be used as a
context manager and will swallow expected failures, as identified by
substrings of the error message Mercurial emits."""
try:
yield
h2, l2, s2 = colorsys.rgb_to_hls(r2, g2, b2)
self.assertColorsValid(h=(h, h2), l=(l, l2), s=(s, s2))
@unittest.expectedFailure
@given(r=st.floats(0, 1), g=st.floats(0, 1), b=st.floats(0, 1))
def test_rgb_hsv_round_trip(self, r, g, b):
h, s, v = colorsys.rgb_to_hsv(r, g, b)
r2, g2, b2 = colorsys.hsv_to_rgb(h, s, v)
self.assertColorsValid(r=(r, r2), g=(g, g2), b=(b, b2))
h2, s2, v2 = colorsys.rgb_to_hls(r2, g2, b2)
self.assertColorsValid(h=(h, h2), s=(s, s2), v=(v, v2))
text_strategy = st.text(alphabet=st.characters(
blacklist_categories=["Cc", "Cs"]))
plistlib_data = st.recursive(
st.booleans()
| st.binary()
| st.datetimes().map(lambda d: d.replace(microsecond=0))
| st.integers(min_value=-1 << 63, max_value=1 << 64 - 1)
| st.floats(allow_nan=False)
| text_strategy,
lambda sub_strategy: st.lists(sub_strategy)
| st.dictionaries(text_strategy, sub_strategy),
)
class TestPlistlib(unittest.TestCase):
@settings(suppress_health_check=HealthCheck.all())
""" Tests for the head() utility function """
import random
import string
import pytest
from hypothesis import given, strategies as st
from pypiper.utils import head
__author__ = "Vince Reuter"
__email__ = "*****@*****.**"
NUMBERS_AND_LETTERS = list(string.ascii_letters) + list(range(-9, 10))
# Strategy for generating a pretty arbitrary atomic
ATOMICS = st.deferred(lambda: st.booleans() | st.characters() | st.integers() |
st.floats(allow_nan=False) | st.text())
def pytest_generate_tests(metafunc):
""" Test case generation/parameterization for this module. """
if "seqtype" in metafunc.fixturenames:
metafunc.parametrize("seqtype", [tuple, list])
if "iter_cast" in metafunc.fixturenames:
metafunc.parametrize("iter_cast", [lambda c: c, lambda c: iter(c)])
if "h" in metafunc.fixturenames and "xs" in metafunc.fixturenames:
metafunc.parametrize(["h", "xs"],
[(random.choice(NUMBERS_AND_LETTERS), [
random.choice(NUMBERS_AND_LETTERS)
for _ in range(random.randint(5, 10))
]) for _ in range(10)])
assert locations == expected_locations
def test_extract_title_with_italics():
xml = '<article><article-title>Activating mutations in <italic>ALK</italic> provide a therapeutic target in neuroblastoma</article-title></article>'
chunks = extract_text_chunks([etree.fromstring(xml)])
assert len(chunks) == 1
assert (
'Activating mutations in ALK provide a therapeutic target in neuroblastoma'
== chunks[0].text
)
@given(
values=st.lists(
st.text(alphabet=st.characters(blacklist_categories=['Cc', 'Cs'])), min_size=1, max_size=50
),
rows=st.integers(min_value=1, max_value=3),
cols=st.integers(min_value=1, max_value=3),
)
def test_extract_delimited_table(values: List[str or int or float or None], rows: int, cols: int):
values = [escape(v) for v in values]
rows_xml = []
values_used = set()
for row_index in range(rows):
tr = []
for col_index in range(cols):
value = values[(row_index * col_index + col_index) % len(values)]
tr.append(
f'\n<td rowspan="1" colspan="1" id="cell_{row_index}_{col_index}">{value}</td>'
# the character encoding scheme used to convert the user-pass into an
# octet sequence. In practice, most implementations chose either a
# locale-specific encoding such as ISO-8859-1 ([ISO-8859-1]), or UTF-8
# ([RFC3629]). For backwards compatibility reasons, this specification
# continues to leave the default encoding undefined, as long as it is
# compatible with US-ASCII (mapping any US-ASCII character to a single
# octet matching the US-ASCII character code).
#
# In particular, Firefox still does *very* special things if you provide
# non-BMP characters in a username or password.
#
# There's not a lot we can do about this so we are going to assume UTF-8
# encoding for the user-pass string, and these tests verify that we
# successfully decode valid Unicode user-pass strings.
#
VALID_USERNAME_CHARS = st.characters(
blacklist_characters=INVALID_USERNAME_CHARS)
VALID_PASSWORD_CHARS = st.characters(
blacklist_characters=INVALID_USER_PASS_CHARS)
class TestBasicAuthCreds(object):
@given(username=st.text(alphabet=VALID_USERNAME_CHARS),
password=st.text(alphabet=VALID_PASSWORD_CHARS))
def test_valid(self, username, password, pyramid_request):
user_pass = username + ':' + password
creds = ('Basic', base64.standard_b64encode(user_pass.encode('utf-8')))
pyramid_request.authorization = creds
assert util.basic_auth_creds(pyramid_request) == (username, password)
def test_missing(self, pyramid_request):
'Discovering collections for pair foobar'
])
result = runner.invoke(['sync'])
assert not result.exception
assert set(result.output.splitlines()) == set([
'Syncing bambaz',
'Syncing foobar',
])
@given(collections=st.sets(
st.text(
st.characters(
blacklist_characters=set(
u'./\x00' # Invalid chars on POSIX filesystems
),
# Surrogates can't be encoded to utf-8 in Python
blacklist_categories=set(['Cs'])),
min_size=1,
max_size=50),
min_size=1))
@example(collections=[u'persönlich'])
@example(collections=set('aA'))
def test_create_collections(subtest, collections):
@subtest
def test_inner(tmpdir, runner):
runner.write_with_general(
dedent('''
[pair foobar]
a = "foo"
b = "bar"
# Configure Hypothesis to run faster when iterating locally
settings.register_profile(
"dev", settings(default_settings, max_examples=5, timeout=0))
# ... and use the defaults (which have more combinations) when running
# on CI, which we want to be more deterministic.
settings.register_profile(
"ci", settings(default_settings, derandomize=True, timeout=0))
# Use the dev profile by default, but use the ci profile on sandcastle.
settings.load_profile(
"ci" if is_sandcastle() else os.getenv("HYPOTHESIS_PROFILE", "dev"))
# Some helpers for Hypothesis decorators
FILENAME_STRATEGY = st.text(
st.characters(min_codepoint=1,
max_codepoint=1000,
blacklist_characters="/:\\"),
min_size=1,
)
# We need to set a global (but non-conflicting) path to store some state
# during hypothesis example runs. We want to avoid putting this state in
# the repo.
set_hypothesis_home_dir(tempfile.mkdtemp(prefix="eden_hypothesis."))
atexit.register(shutil.rmtree, hypothesis_home_dir())
if not edenclient.can_run_eden():
# This is avoiding a reporting noise issue in our CI that files
# tasks about skipped tests. Let's just skip defining most of them
# to avoid the noise if we know that they won't work anyway.
TestParent = typing.cast(Type[unittest.TestCase], object)
@pytest.mark.parametrize(
"file_data, shape, expected_value",
[
("PROP\n 1 5 3 7 2 6 4 8 /\n", (2, 2, 2), [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]),
("PROP\n 1 3 2 4 /\n", (1, 2, 2), [[[1, 2], [3, 4]]]),
],
)
def test_read_grdecl_3d_property(file_data, shape, expected_value):
with patch("builtins.open", mock_open(read_data=file_data)) as mock_file:
np.array_equal(
read_grdecl_3d_property(mock_file, "PROP", shape, int), expected_value
)
keywords = st.text(
alphabet=st.characters(whitelist_categories=("Nd", "Lu")), min_size=1
)
grid_properties = arrays(
elements=st.floats(), dtype="float", shape=st.tuples(indecies, indecies, indecies)
)
@given(keywords, grid_properties)
def test_read_write_grid_property_is_identity(keyword, grid_property):
values = [str(v) for v in grid_property.flatten(order="F")]
file_data = f"{keyword}\n {' '.join(values)} /"
with patch("builtins.open", mock_open(read_data=file_data)) as mock_file:
assert_allclose(
read_grdecl_3d_property(mock_file, keyword, grid_property.shape),
grid_property,
)
def test_when_nothing_could_be_produced():
with pytest.raises(InvalidArgument):
characters(whitelist_categories=["Cc"],
min_codepoint=ord("0"),
max_codepoint=ord("9")).example()
BEGIN:DAYLIGHT
TZOFFSETFROM:+0100
TZOFFSETTO:+0200
TZNAME:CEST
DTSTART:19700329T020000
RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3
END:DAYLIGHT
BEGIN:STANDARD
TZOFFSETFROM:+0200
TZOFFSETTO:+0100
TZNAME:CET
DTSTART:19701025T030000
RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
END:STANDARD
END:VTIMEZONE
""" + BARE_EVENT_TEMPLATE + """
END:VCALENDAR""")
SIMPLE_TEMPLATE = """BEGIN:FOO
UID:{uid}
X-SOMETHING:{r}
HAHA:YES
END:FOO"""
printable_characters_strategy = st.text(
st.characters(blacklist_categories=("Cc", "Cs")))
uid_strategy = st.text(st.characters(blacklist_categories=("Zs", "Zl", "Zp",
"Cc", "Cs")),
min_size=1).filter(lambda x: x.strip() == x)
from typing import Union
# Non-standard library python package imports
from hypothesis import given
from hypothesis import strategies as st
# Imports of module(s) internal to this project/package
from rosahelikopter import repository_is_relevant_for_overview
from rosahelikopter.markdown import make_markdown_table
MARKDOWN_VALID_TABLE_ROWS_REGEX_PATTERN = r'\| \[[-\w\d]+/[-\w\d]+\]\(.*\) \| .+ \|'
GITHUB_NAME_WITH_OWNER = r'[\w\d-]+/[\w\d-]+'
HYPOTHESIS_GITHUB_NAMES_CHARACTERS = st.characters(
whitelist_categories=(
# https://en.wikipedia.org/wiki/Unicode_character_property#General_Category
'Lu',
'Ll',
'Nd',
), )
# https://stackoverflow.com/a/1151705/1503549
class hashabledict(dict):
def __hash__(self):
return hash(tuple(sorted(self.items())))
@given(
st.lists(
st.fixed_dictionaries(
hashabledict(
assert b'\xa0'.decode('pdfdoc') == '€'
def test_unicode_surrogate():
with pytest.raises(ValueError, match=r'surrogate'):
'\ud800'.encode('pdfdoc')
@given(binary())
def test_codec_involution(b):
# For all binary strings, there is a pdfdoc decoding. The encoding of that
# decoding recovers the initial string. (However, not all str have a pdfdoc
# encoding.)
assert b.decode('pdfdoc').encode('pdfdoc') == b
@given(characters())
def test_break_encode(c):
try:
encoded_bytes = c.encode('pdfdoc')
except ValueError as e:
allowed_errors = [
"'pdfdoc' codec can't process Unicode surrogates",
"'pdfdoc' codec can't encode some characters",
]
if any((allowed in str(e)) for allowed in allowed_errors):
return
raise
else:
assert encoded_bytes.decode('pdfdoc') == c
def _strategy(codes, context, is_unicode):
"""Convert SRE regex parse tree to strategy that generates strings matching
that regex represented by that parse tree.
`codes` is either a list of SRE regex elements representations or a
particular element representation. Each element is a tuple of element code
(as string) and parameters. E.g. regex 'ab[0-9]+' compiles to following
elements:
[
(LITERAL, 97),
(LITERAL, 98),
(MAX_REPEAT, (1, 4294967295, [
(IN, [
(RANGE, (48, 57))
])
]))
]
The function recursively traverses regex element tree and converts each
element to strategy that generates strings that match that element.
Context stores
1. List of groups (for backreferences)
2. Active regex flags (e.g. IGNORECASE, DOTALL, UNICODE, they affect
behavior of various inner strategies)
"""
def recurse(codes):
return _strategy(codes, context, is_unicode)
if is_unicode:
empty = u''
to_char = hunichr
else:
empty = b''
to_char = int_to_byte
binary_char = st.binary(min_size=1, max_size=1)
if not isinstance(codes, tuple):
# List of codes
strategies = []
i = 0
while i < len(codes):
if codes[i][0] == sre.LITERAL and \
not context.flags & re.IGNORECASE:
# Merge subsequent "literals" into one `just()` strategy
# that generates corresponding text if no IGNORECASE
j = i + 1
while j < len(codes) and codes[j][0] == sre.LITERAL:
j += 1
if i + 1 < j:
strategies.append(st.just(
empty.join([to_char(charcode)
for (_, charcode) in codes[i:j]])
))
i = j
continue
strategies.append(recurse(codes[i]))
i += 1
# We handle this separately at the top level, but some regex can
# contain empty lists internally, so we need to handle this here too.
if not strategies:
return st.just(empty)
if len(strategies) == 1:
return strategies[0]
return st.tuples(*strategies).map(empty.join)
else:
# Single code
code, value = codes
if code == sre.LITERAL:
# Regex 'a' (single char)
c = to_char(value)
if context.flags & re.IGNORECASE and \
re.match(c, c.swapcase(), re.IGNORECASE) is not None:
# We do the explicit check for swapped-case matching because
# eg 'ß'.upper() == 'SS' and ignorecase doesn't match it.
return st.sampled_from([c, c.swapcase()])
return st.just(c)
elif code == sre.NOT_LITERAL:
# Regex '[^a]' (negation of a single char)
c = to_char(value)
blacklist = set(c)
if context.flags & re.IGNORECASE and \
re.match(c, c.swapcase(), re.IGNORECASE) is not None:
blacklist |= set(c.swapcase())
if is_unicode:
return st.characters(blacklist_characters=blacklist)
else:
return binary_char.filter(lambda c: c not in blacklist)
elif code == sre.IN:
# Regex '[abc0-9]' (set of characters)
negate = value[0][0] == sre.NEGATE
if is_unicode:
builder = CharactersBuilder(negate, context.flags)
else:
builder = BytesBuilder(negate, context.flags)
for charset_code, charset_value in value:
if charset_code == sre.NEGATE:
# Regex '[^...]' (negation)
# handled by builder = CharactersBuilder(...) above
pass
elif charset_code == sre.LITERAL:
# Regex '[a]' (single char)
builder.add_char(charset_value)
elif charset_code == sre.RANGE:
# Regex '[a-z]' (char range)
low, high = charset_value
for char_code in hrange(low, high + 1):
builder.add_char(char_code)
elif charset_code == sre.CATEGORY:
# Regex '[\w]' (char category)
builder.add_category(charset_value)
else: # pragma: no cover
# Currently there are no known code points other than
# handled here. This code is just future proofing
raise AssertionError('Unknown charset code: %s'
% charset_code)
return builder.strategy
elif code == sre.ANY:
# Regex '.' (any char)
if is_unicode:
if context.flags & re.DOTALL:
return st.characters()
return st.characters(blacklist_characters=u'\n')
else:
if context.flags & re.DOTALL:
return binary_char
return binary_char.filter(lambda c: c != b'\n')
elif code == sre.AT:
# Regexes like '^...', '...$', '\bfoo', '\Bfoo'
# An empty string (or newline) will match the token itself, but
# we don't and can't check the position (eg '%' at the end)
return st.just(empty)
elif code == sre.SUBPATTERN:
# Various groups: '(...)', '(:...)' or '(?P<name>...)'
old_flags = context.flags
if HAS_SUBPATTERN_FLAGS: # pragma: no cover
# This feature is available only in specific Python versions
context.flags = (context.flags | value[1]) & ~value[2]
strat = _strategy(value[-1], context, is_unicode)
context.flags = old_flags
if value[0]:
strat = update_group(value[0], strat)
return strat
elif code == sre.GROUPREF:
# Regex '\\1' or '(?P=name)' (group reference)
return reuse_group(value)
elif code == sre.ASSERT:
# Regex '(?=...)' or '(?<=...)' (positive lookahead/lookbehind)
return recurse(value[1])
elif code == sre.ASSERT_NOT:
# Regex '(?!...)' or '(?<!...)' (negative lookahead/lookbehind)
return st.just(empty)
elif code == sre.BRANCH:
# Regex 'a|b|c' (branch)
return st.one_of([recurse(branch) for branch in value[1]])
elif code in [sre.MIN_REPEAT, sre.MAX_REPEAT]:
# Regexes 'a?', 'a*', 'a+' and their non-greedy variants
# (repeaters)
at_least, at_most, subregex = value
if at_most == sre.MAXREPEAT:
at_most = None
if at_least == 0 and at_most == 1:
return st.just(empty) | recurse(subregex)
return st.lists(recurse(subregex),
min_size=at_least,
max_size=at_most).map(empty.join)
elif code == sre.GROUPREF_EXISTS:
# Regex '(?(id/name)yes-pattern|no-pattern)'
# (if group exists choice)
return group_conditional(
value[0],
recurse(value[1]),
recurse(value[2]) if value[2] else st.just(empty),
)
else: # pragma: no cover
# Currently there are no known code points other than handled here.
# This code is just future proofing
raise AssertionError('Unknown code point: %s' % repr(code))
def test_parse_with_odd_quotes_combinations(query_in, query_out):
assert parser.parse(query_in) == query_out
@given(st.text())
@pytest.mark.fuzz
def test_parse_always_return_a_multidict(text):
"""Given any string input, output should always be a MultiDict."""
result = parser.parse(text)
assert isinstance(result, MultiDict)
# Combinations of strings containing any number of quotes are already tested
# separately.
char_blacklist = parser.whitespace.union(set('\'"'))
nonwhitespace_chars = st.characters(blacklist_characters=char_blacklist)
nonwhitespace_text = st.text(alphabet=nonwhitespace_chars, min_size=1)
@given(kw=st.sampled_from(parser.named_fields), value=nonwhitespace_text)
@pytest.mark.fuzz
def test_parse_with_any_nonwhitespace_text(kw, value):
result = parser.parse(kw + ':' + value)
assert result.get(kw) == value
@pytest.mark.parametrize(
"query",
[
# Plain dictionary
{
"Discovering collections for pair foobar",
}
result = runner.invoke(["sync"])
assert not result.exception
assert set(result.output.splitlines()) == {
"Syncing bambaz",
"Syncing foobar",
}
collections_strategy = st.sets(
st.text(
st.characters(
blacklist_characters=set(
"./\x00"), # Invalid chars on POSIX filesystems
# Surrogates can't be encoded to utf-8 in Python
blacklist_categories={"Cs"},
),
min_size=1,
max_size=50,
),
min_size=1,
)
# XXX: https://github.com/pimutils/vdirsyncer/issues/617
@pytest.mark.skipif(sys.platform == "darwin",
reason="This test inexplicably fails")
@pytest.mark.parametrize(
"collections",
[
class GLib2PythonWriterTest(unittest.TestCase):
def setUp(self):
logger.addHandler(qhandler)
def tearDown(self):
was_empty = q.empty()
while not q.empty():
q.get(timeout=1)
self.assertTrue(was_empty)
logger.removeHandler(qhandler)
def _register(self, l):
GLib.log_set_writer_func(l.logWriterFunc, None)
def _log(self, logger_name, flags, message, fields=None):
f = {'MESSAGE': GLib.Variant('s', message)}
if fields is not None:
f.update(fields)
GLib.log_variant(logger_name, flags, GLib.Variant('a{sv}', f))
@given(
strategies.text(alphabet=strategies.characters(
blacklist_categories=('C'), blacklist_characters='\x00'),
min_size=1))
def test_basic(self, msg):
self._register(g2p.Logger())
self._log(LOGGER_NAME, GLib.LogLevelFlags.LEVEL_WARNING, msg)
record = q.get(timeout=1)
self.assertEqual(record.message, msg)
@given(
strategies.sampled_from([
# (GLib.LogLevelFlags.LEVEL_ERROR, logging.ERROR), # fatal, so aborts
(GLib.LogLevelFlags.LEVEL_CRITICAL, logging.CRITICAL),
(GLib.LogLevelFlags.LEVEL_WARNING, logging.WARNING),
(GLib.LogLevelFlags.LEVEL_MESSAGE, logging.INFO),
(GLib.LogLevelFlags.LEVEL_INFO, logging.INFO),
(GLib.LogLevelFlags.LEVEL_DEBUG, logging.DEBUG),
]))
def test_loglevels(self, m):
self._register(g2p.Logger())
glib_level, logging_level = m
self._log(LOGGER_NAME, m[0], "some_message")
self.assertEqual(m[1], q.get(timeout=1).levelno)
@given(
strategies.sampled_from([
# (GLib.LogLevelFlags.LEVEL_ERROR, logging.ERROR), # fatal, so aborts
(GLib.LogLevelFlags.LEVEL_CRITICAL, logging.CRITICAL),
(GLib.LogLevelFlags.LEVEL_WARNING, logging.WARNING),
(GLib.LogLevelFlags.LEVEL_MESSAGE, logging.INFO),
(GLib.LogLevelFlags.LEVEL_INFO, logging.INFO),
(GLib.LogLevelFlags.LEVEL_DEBUG, logging.DEBUG),
]))
def test_loglevels_priority(self, m):
self._register(g2p.Logger(use_priority_field=False))
self._log(LOGGER_NAME, m[0], "some_message")
self.assertEqual(m[1], q.get(timeout=1).levelno)
@given(
strategies.lists(strategies.text(alphabet=strategies.characters(
blacklist_categories=('C'), blacklist_characters='\x00'),
min_size=1),
min_size=1))
def test_domain(self, domain):
self._register(g2p.Logger())
level = GLib.LogLevelFlags.LEVEL_INFO
domain = '-'.join(domain)
logger = logging.getLogger(LOGGER_NAME + '.' +
domain.replace('-', '.'))
lq = queue.Queue()
logger.addHandler(logging.handlers.QueueHandler(lq))
self._log(LOGGER_NAME + '-' + domain, level, domain)
self.assertEqual(domain, lq.get(timeout=1).message)
self.assertTrue(lq.empty())
while not q.empty():
q.get(timeout=1)
@given(
strategies.lists(strategies.from_regex('[a-zA-Z][a-zA-Z0-9]*',
fullmatch=True),
min_size=1,
max_size=5))
def test_domain_prefix(self, domain):
self._register(g2p.Logger(logger_prefix=LOGGER_NAME + '.'))
level = GLib.LogLevelFlags.LEVEL_INFO
domain = '-'.join(domain)
logger = logging.getLogger(LOGGER_NAME + '.' +
domain.replace('-', '.'))
lq = queue.Queue()
handler = logging.handlers.QueueHandler(lq)
logger.addHandler(handler)
try:
self._log(domain, level, domain)
self.assertEqual(domain, lq.get(timeout=1).message)
self.assertTrue(lq.empty())
finally:
logger.removeHandler(handler)
while not q.empty():
q.get(timeout=1)
def test_invalid_message(self):
self._register(g2p.Logger())
self._log(
LOGGER_NAME,
GLib.LogLevelFlags.LEVEL_WARNING,
'',
fields={
'MESSAGE': GLib.Variant('ay', b'\xFF') # Not valid UTF-8
})
record = q.get(timeout=1)
self.assertEqual(record.message, '\uFFFD') # At least still a string
self.assertTrue(q.empty())
class TestEtcd3(object):
class MockedException(grpc.RpcError):
def __init__(self, code):
self._code = code
def code(self):
return self._code
@pytest.fixture
def etcd(self):
endpoint = os.environ.get('PYTHON_ETCD_HTTP_URL')
timeout = 5
if endpoint:
url = urlparse(endpoint)
with etcd3.client(host=url.hostname,
port=url.port,
timeout=timeout) as client:
yield client
else:
with etcd3.client() as client:
yield client
@retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
def delete_keys_definitely():
# clean up after fixture goes out of scope
etcdctl('del', '--prefix', '/')
out = etcdctl('get', '--prefix', '/')
assert 'kvs' not in out
delete_keys_definitely()
def test_get_unknown_key(self, etcd):
value, meta = etcd.get('probably-invalid-key')
assert value is None
assert meta is None
@given(characters(blacklist_categories=['Cs', 'Cc']))
def test_get_key(self, etcd, string):
etcdctl('put', '/doot/a_key', string)
returned, _ = etcd.get('/doot/a_key')
assert returned == string.encode('utf-8')
@given(characters(blacklist_categories=['Cs', 'Cc']))
def test_get_random_key(self, etcd, string):
etcdctl('put', '/doot/' + string, 'dootdoot')
returned, _ = etcd.get('/doot/' + string)
assert returned == b'dootdoot'
@given(
characters(blacklist_categories=['Cs', 'Cc']),
characters(blacklist_categories=['Cs', 'Cc']),
)
def test_get_key_serializable(self, etcd, key, string):
etcdctl('put', '/doot/' + key, string)
with _out_quorum():
returned, _ = etcd.get('/doot/' + key, serializable=True)
assert returned == string.encode('utf-8')
@given(characters(blacklist_categories=['Cs', 'Cc']))
def test_get_have_cluster_revision(self, etcd, string):
etcdctl('put', '/doot/' + string, 'dootdoot')
_, md = etcd.get('/doot/' + string)
assert md.response_header.revision > 0
@given(characters(blacklist_categories=['Cs', 'Cc']))
def test_put_key(self, etcd, string):
etcd.put('/doot/put_1', string)
out = etcdctl('get', '/doot/put_1')
assert base64.b64decode(out['kvs'][0]['value']) == \
string.encode('utf-8')
@given(characters(blacklist_categories=['Cs', 'Cc']))
def test_put_has_cluster_revision(self, etcd, string):
response = etcd.put('/doot/put_1', string)
assert response.header.revision > 0
@given(characters(blacklist_categories=['Cs', 'Cc']))
def test_put_has_prev_kv(self, etcd, string):
etcdctl('put', '/doot/put_1', 'old_value')
response = etcd.put('/doot/put_1', string, prev_kv=True)
assert response.prev_kv.value == b'old_value'
@given(characters(blacklist_categories=['Cs', 'Cc']))
def test_put_if_not_exists(self, etcd, string):
txn_status = etcd.put_if_not_exists('/doot/put_1', string)
assert txn_status is True
txn_status = etcd.put_if_not_exists('/doot/put_1', string)
assert txn_status is False
etcdctl('del', '/doot/put_1')
def test_delete_key(self, etcd):
etcdctl('put', '/doot/delete_this', 'delete pls')
v, _ = etcd.get('/doot/delete_this')
assert v == b'delete pls'
deleted = etcd.delete('/doot/delete_this')
assert deleted is True
deleted = etcd.delete('/doot/delete_this')
assert deleted is False
deleted = etcd.delete('/doot/not_here_dude')
assert deleted is False
v, _ = etcd.get('/doot/delete_this')
assert v is None
def test_delete_has_cluster_revision(self, etcd):
response = etcd.delete('/doot/delete_this', return_response=True)
assert response.header.revision > 0
def test_delete_has_prev_kv(self, etcd):
etcdctl('put', '/doot/delete_this', 'old_value')
response = etcd.delete('/doot/delete_this',
prev_kv=True,
return_response=True)
assert response.prev_kvs[0].value == b'old_value'
def test_delete_keys_with_prefix(self, etcd):
etcdctl('put', '/foo/1', 'bar')
etcdctl('put', '/foo/2', 'baz')
v, _ = etcd.get('/foo/1')
assert v == b'bar'
v, _ = etcd.get('/foo/2')
assert v == b'baz'
response = etcd.delete_prefix('/foo')
assert response.deleted == 2
v, _ = etcd.get('/foo/1')
assert v is None
v, _ = etcd.get('/foo/2')
assert v is None
def test_new_watch_error(self, etcd):
# Trigger a failure while waiting on the new watch condition
with mock.patch.object(etcd.watcher._new_watch_cond,
'wait',
side_effect=ValueError):
with pytest.raises(ValueError):
etcd.watch('/foo')
# Ensure a new watch can be created
events, cancel = etcd.watch('/foo')
etcdctl('put', '/foo', '42')
next(events)
cancel()
def test_watch_key(self, etcd):
def update_etcd(v):
etcdctl('put', '/doot/watch', v)
out = etcdctl('get', '/doot/watch')
assert base64.b64decode(out['kvs'][0]['value']) == \
utils.to_bytes(v)
def update_key():
# sleep to make watch can get the event
time.sleep(3)
update_etcd('0')
time.sleep(1)
update_etcd('1')
time.sleep(1)
update_etcd('2')
time.sleep(1)
update_etcd('3')
time.sleep(1)
t = threading.Thread(name="update_key", target=update_key)
t.start()
change_count = 0
events_iterator, cancel = etcd.watch(b'/doot/watch')
for event in events_iterator:
assert event.key == b'/doot/watch'
assert event.value == \
utils.to_bytes(str(change_count))
# if cancel worked, we should not receive event 3
assert event.value != utils.to_bytes('3')
change_count += 1
if change_count > 2:
# if cancel not work, we will block in this for-loop forever
cancel()
t.join()
def test_watch_key_with_revision_compacted(self, etcd):
etcdctl('put', '/random', '1') # Some data to compact
def update_etcd(v):
etcdctl('put', '/watchcompation', v)
out = etcdctl('get', '/watchcompation')
assert base64.b64decode(out['kvs'][0]['value']) == \
utils.to_bytes(v)
def update_key():
# sleep to make watch can get the event
time.sleep(3)
update_etcd('0')
time.sleep(1)
update_etcd('1')
time.sleep(1)
update_etcd('2')
time.sleep(1)
update_etcd('3')
time.sleep(1)
t = threading.Thread(name="update_key", target=update_key)
t.start()
def watch_compacted_revision_test():
events_iterator, cancel = etcd.watch(b'/watchcompation',
start_revision=1)
error_raised = False
compacted_revision = 0
try:
next(events_iterator)
except Exception as err:
error_raised = True
assert isinstance(err, etcd3.exceptions.RevisionCompactedError)
compacted_revision = err.compacted_revision
assert error_raised is True
assert compacted_revision == 2
change_count = 0
events_iterator, cancel = etcd.watch(
b'/watchcompation', start_revision=compacted_revision)
for event in events_iterator:
assert event.key == b'/watchcompation'
assert event.value == \
utils.to_bytes(str(change_count))
# if cancel worked, we should not receive event 3
assert event.value != utils.to_bytes('3')
change_count += 1
if change_count > 2:
cancel()
# Compact etcd and test watcher
etcd.compact(2)
watch_compacted_revision_test()
t.join()
def test_watch_exception_during_watch(self, etcd):
def pass_exception_to_callback(callback):
time.sleep(1)
callback(self.MockedException(grpc.StatusCode.UNAVAILABLE))
def add_callback_mock(*args, **kwargs):
callback = args[1]
t = threading.Thread(name="pass_exception_to_callback",
target=pass_exception_to_callback,
args=[callback])
t.start()
return 1
watcher_mock = mock.MagicMock()
watcher_mock.add_callback = add_callback_mock
etcd.watcher = watcher_mock
events_iterator, cancel = etcd.watch('foo')
with pytest.raises(etcd3.exceptions.ConnectionFailedError):
for _ in events_iterator:
pass
def test_watch_timeout_on_establishment(self, etcd):
foo_etcd = etcd3.client(timeout=3)
def slow_watch_mock(*args, **kwargs):
time.sleep(4)
foo_etcd.watcher._watch_stub.Watch = slow_watch_mock # noqa
with pytest.raises(etcd3.exceptions.WatchTimedOut):
foo_etcd.watch('foo')
def test_watch_prefix(self, etcd):
def update_etcd(v):
etcdctl('put', '/doot/watch/prefix/' + v, v)
out = etcdctl('get', '/doot/watch/prefix/' + v)
assert base64.b64decode(out['kvs'][0]['value']) == \
utils.to_bytes(v)
def update_key():
# sleep to make watch can get the event
time.sleep(3)
update_etcd('0')
time.sleep(1)
update_etcd('1')
time.sleep(1)
update_etcd('2')
time.sleep(1)
update_etcd('3')
time.sleep(1)
t = threading.Thread(name="update_key_prefix", target=update_key)
t.start()
change_count = 0
events_iterator, cancel = etcd.watch_prefix('/doot/watch/prefix/')
for event in events_iterator:
assert event.key == \
utils.to_bytes('/doot/watch/prefix/{}'.format(change_count))
assert event.value == \
utils.to_bytes(str(change_count))
# if cancel worked, we should not receive event 3
assert event.value != utils.to_bytes('3')
change_count += 1
if change_count > 2:
# if cancel not work, we will block in this for-loop forever
cancel()
t.join()
def test_watch_prefix_callback(self, etcd):
def update_etcd(v):
etcdctl('put', '/doot/watch/prefix/callback/' + v, v)
out = etcdctl('get', '/doot/watch/prefix/callback/' + v)
assert base64.b64decode(out['kvs'][0]['value']) == \
utils.to_bytes(v)
def update_key():
# sleep to make watch can get the event
time.sleep(3)
update_etcd('0')
time.sleep(1)
update_etcd('1')
time.sleep(1)
events = []
def callback(event):
events.extend(event.events)
t = threading.Thread(name="update_key_prefix", target=update_key)
t.start()
watch_id = etcd.add_watch_prefix_callback(
'/doot/watch/prefix/callback/', callback)
t.join()
etcd.cancel_watch(watch_id)
assert len(events) == 2
assert events[0].key.decode() == '/doot/watch/prefix/callback/0'
assert events[0].value.decode() == '0'
assert events[1].key.decode() == '/doot/watch/prefix/callback/1'
assert events[1].value.decode() == '1'
def test_sequential_watch_prefix_once(self, etcd):
try:
etcd.watch_prefix_once('/doot/', 1)
except etcd3.exceptions.WatchTimedOut:
pass
try:
etcd.watch_prefix_once('/doot/', 1)
except etcd3.exceptions.WatchTimedOut:
pass
try:
etcd.watch_prefix_once('/doot/', 1)
except etcd3.exceptions.WatchTimedOut:
pass
def test_watch_responses(self, etcd):
# Test watch_response & watch_once_response
revision = etcd.put('/doot/watch', '0').header.revision
etcd.put('/doot/watch', '1')
responses_iterator, cancel = \
etcd.watch_response('/doot/watch', start_revision=revision)
response_1 = next(responses_iterator)
cancel()
response_2 = etcd.watch_once_response('/doot/watch',
start_revision=revision)
for response in [response_1, response_2]:
count = 0
# check that the response contains the etcd revision
assert response.header.revision > 0
assert len(response.events) == 2
for event in response.events:
assert event.key == b'/doot/watch'
assert event.value == utils.to_bytes(str(count))
count += 1
# Test watch_prefix_response & watch_prefix_once_response
success_ops = [
etcd.transactions.put('/doot/watch/prefix/0', '0'),
etcd.transactions.put('/doot/watch/prefix/1', '1')
]
revision = etcd.transaction([], success_ops,
[])[1][0].response_put.header.revision
responses_iterator, cancel = \
etcd.watch_prefix_response('/doot/watch/prefix/',
start_revision=revision)
response_1 = next(responses_iterator)
cancel()
response_2 = etcd.watch_prefix_once_response('/doot/watch/prefix/',
start_revision=revision)
for response in [response_1, response_2]:
count = 0
assert response.header.revision == revision
assert len(response.events) == 2
for event in response.events:
assert event.key == \
utils.to_bytes('/doot/watch/prefix/{}'.format(count))
assert event.value == utils.to_bytes(str(count))
count += 1
def test_transaction_success(self, etcd):
etcdctl('put', '/doot/txn', 'dootdoot')
etcd.transaction(
compare=[etcd.transactions.value('/doot/txn') == 'dootdoot'],
success=[etcd.transactions.put('/doot/txn', 'success')],
failure=[etcd.transactions.put('/doot/txn', 'failure')])
out = etcdctl('get', '/doot/txn')
assert base64.b64decode(out['kvs'][0]['value']) == b'success'
def test_transaction_failure(self, etcd):
etcdctl('put', '/doot/txn', 'notdootdoot')
etcd.transaction(
compare=[etcd.transactions.value('/doot/txn') == 'dootdoot'],
success=[etcd.transactions.put('/doot/txn', 'success')],
failure=[etcd.transactions.put('/doot/txn', 'failure')])
out = etcdctl('get', '/doot/txn')
assert base64.b64decode(out['kvs'][0]['value']) == b'failure'
def test_ops_to_requests(self, etcd):
with pytest.raises(Exception):
etcd._ops_to_requests(['not_transaction_type'])
with pytest.raises(TypeError):
etcd._ops_to_requests(0)
@pytest.mark.skipif(etcd_version < 'v3.3',
reason="requires etcd v3.3 or higher")
def test_nested_transactions(self, etcd):
etcd.transaction(
compare=[],
success=[
etcd.transactions.put('/doot/txn1', '1'),
etcd.transactions.txn(
compare=[],
success=[etcd.transactions.put('/doot/txn2', '2')],
failure=[])
],
failure=[])
value, _ = etcd.get('/doot/txn1')
assert value == b'1'
value, _ = etcd.get('/doot/txn2')
assert value == b'2'
@pytest.mark.skipif(etcd_version < 'v3.3',
reason="requires etcd v3.3 or higher")
def test_transaction_range_conditions(self, etcd):
etcdctl('put', '/doot/key1', 'dootdoot')
etcdctl('put', '/doot/key2', 'notdootdoot')
range_end = utils.increment_last_byte(utils.to_bytes('/doot/'))
compare = [etcd.transactions.value('/doot/', range_end) == 'dootdoot']
status, _ = etcd.transaction(compare=compare, success=[], failure=[])
assert not status
etcdctl('put', '/doot/key2', 'dootdoot')
status, _ = etcd.transaction(compare=compare, success=[], failure=[])
assert status
def test_replace_success(self, etcd):
etcd.put('/doot/thing', 'toot')
status = etcd.replace('/doot/thing', 'toot', 'doot')
v, _ = etcd.get('/doot/thing')
assert v == b'doot'
assert status is True
def test_replace_fail(self, etcd):
etcd.put('/doot/thing', 'boot')
status = etcd.replace('/doot/thing', 'toot', 'doot')
v, _ = etcd.get('/doot/thing')
assert v == b'boot'
assert status is False
def test_get_prefix(self, etcd):
for i in range(20):
etcdctl('put', '/doot/range{}'.format(i), 'i am a range')
for i in range(5):
etcdctl('put', '/doot/notrange{}'.format(i), 'i am a not range')
values = list(etcd.get_prefix('/doot/range'))
assert len(values) == 20
for value, _ in values:
assert value == b'i am a range'
def test_get_prefix_keys_only(self, etcd):
for i in range(20):
etcdctl('put', '/doot/range{}'.format(i), 'i am a range')
for i in range(5):
etcdctl('put', '/doot/notrange{}'.format(i), 'i am a not range')
values = list(etcd.get_prefix('/doot/range', keys_only=True))
assert len(values) == 20
for value, meta in values:
assert meta.key.startswith(b"/doot/range")
assert not value
def test_get_range(self, etcd):
for char in string.ascii_lowercase:
if char < 'p':
etcdctl('put', '/doot/' + char, 'i am in range')
else:
etcdctl('put', '/doot/' + char, 'i am not in range')
values = list(etcd.get_range('/doot/a', '/doot/p'))
assert len(values) == 15
for value, _ in values:
assert value == b'i am in range'
def test_all_not_found_error(self, etcd):
result = list(etcd.get_all())
assert not result
def test_range_not_found_error(self, etcd):
for i in range(5):
etcdctl('put', '/doot/notrange{}'.format(i), 'i am a not range')
result = list(etcd.get_prefix('/doot/range'))
assert not result
def test_get_all(self, etcd):
for i in range(20):
etcdctl('put', '/doot/range{}'.format(i), 'i am in all')
for i in range(5):
etcdctl('put', '/doot/notrange{}'.format(i), 'i am in all')
values = list(etcd.get_all())
assert len(values) == 25
for value, _ in values:
assert value == b'i am in all'
def test_get_all_keys_only(self, etcd):
for i in range(20):
etcdctl('put', '/doot/range{}'.format(i), 'i am in all')
for i in range(5):
etcdctl('put', '/doot/notrange{}'.format(i), 'i am in all')
values = list(etcd.get_all(keys_only=True))
assert len(values) == 25
for value, meta in values:
assert meta.key.startswith(b"/doot/")
assert not value
def test_sort_order(self, etcd):
def remove_prefix(string, prefix):
return string[len(prefix):]
initial_keys = 'abcde'
initial_values = 'qwert'
for k, v in zip(initial_keys, initial_values):
etcdctl('put', '/doot/{}'.format(k), v)
keys = ''
for value, meta in etcd.get_prefix('/doot', sort_order='ascend'):
keys += remove_prefix(meta.key.decode('utf-8'), '/doot/')
assert keys == initial_keys
reverse_keys = ''
for value, meta in etcd.get_prefix('/doot', sort_order='descend'):
reverse_keys += remove_prefix(meta.key.decode('utf-8'), '/doot/')
assert reverse_keys == ''.join(reversed(initial_keys))
def test_get_response(self, etcd):
etcdctl('put', '/foo/key1', 'value1')
etcdctl('put', '/foo/key2', 'value2')
response = etcd.get_response('/foo/key1')
assert response.header.revision > 0
assert response.count == 1
assert response.kvs[0].key == b'/foo/key1'
assert response.kvs[0].value == b'value1'
response = etcd.get_prefix_response('/foo/', sort_order='ascend')
assert response.header.revision > 0
assert response.count == 2
assert response.kvs[0].key == b'/foo/key1'
assert response.kvs[0].value == b'value1'
assert response.kvs[1].key == b'/foo/key2'
assert response.kvs[1].value == b'value2'
# Test that the response header is accessible even when the
# requested key or range of keys does not exist
etcdctl('del', '--prefix', '/foo/')
response = etcd.get_response('/foo/key1')
assert response.count == 0
assert response.header.revision > 0
response = etcd.get_prefix_response('/foo/')
assert response.count == 0
assert response.header.revision > 0
response = etcd.get_range_response('/foo/key1', '/foo/key3')
assert response.count == 0
assert response.header.revision > 0
response = etcd.get_all_response()
assert response.count == 0
assert response.header.revision > 0
def test_lease_grant(self, etcd):
lease = etcd.lease(1)
assert isinstance(lease.ttl, int_types)
assert isinstance(lease.id, int_types)
def test_lease_revoke(self, etcd):
lease = etcd.lease(1)
lease.revoke()
@pytest.mark.skipif(etcd_version.startswith('v3.0'),
reason="requires etcd v3.1 or higher")
def test_lease_keys_empty(self, etcd):
lease = etcd.lease(1)
assert lease.keys == []
@pytest.mark.skipif(etcd_version.startswith('v3.0'),
reason="requires etcd v3.1 or higher")
def test_lease_single_key(self, etcd):
lease = etcd.lease(1)
etcd.put('/doot/lease_test', 'this is a lease', lease=lease)
assert lease.keys == [b'/doot/lease_test']
@pytest.mark.skipif(etcd_version.startswith('v3.0'),
reason="requires etcd v3.1 or higher")
def test_lease_expire(self, etcd):
key = '/doot/lease_test_expire'
lease = etcd.lease(1)
etcd.put(key, 'this is a lease', lease=lease)
assert lease.keys == [utils.to_bytes(key)]
v, _ = etcd.get(key)
assert v == b'this is a lease'
assert lease.remaining_ttl <= lease.granted_ttl
# wait for the lease to expire
time.sleep(lease.granted_ttl + 2)
v, _ = etcd.get(key)
assert v is None
def test_member_list(self, etcd):
assert len(list(etcd.members)) == 3
for member in etcd.members:
assert member.name.startswith('pifpaf')
for peer_url in member.peer_urls:
assert peer_url.startswith('http://')
for client_url in member.client_urls:
assert client_url.startswith('http://')
assert isinstance(member.id, int_types) is True
def test_lock_acquire(self, etcd):
lock = etcd.lock('lock-1', ttl=10)
assert lock.acquire() is True
assert etcd.get(lock.key)[0] is not None
assert lock.acquire(timeout=0) is False
assert lock.acquire(timeout=1) is False
def test_lock_release(self, etcd):
lock = etcd.lock('lock-2', ttl=10)
assert lock.acquire() is True
assert etcd.get(lock.key)[0] is not None
assert lock.release() is True
v, _ = etcd.get(lock.key)
assert v is None
assert lock.acquire() is True
assert lock.release() is True
assert lock.acquire(timeout=None) is True
def test_lock_expire(self, etcd):
lock = etcd.lock('lock-3', ttl=3)
assert lock.acquire() is True
assert etcd.get(lock.key)[0] is not None
# wait for the lease to expire
time.sleep(9)
v, _ = etcd.get(lock.key)
assert v is None
def test_lock_refresh(self, etcd):
lock = etcd.lock('lock-4', ttl=3)
assert lock.acquire() is True
assert etcd.get(lock.key)[0] is not None
# sleep for the same total time as test_lock_expire, but refresh each
# second
for _ in range(9):
time.sleep(1)
lock.refresh()
assert etcd.get(lock.key)[0] is not None
def test_lock_is_acquired(self, etcd):
lock1 = etcd.lock('lock-5', ttl=2)
assert lock1.is_acquired() is False
lock2 = etcd.lock('lock-5', ttl=2)
lock2.acquire()
assert lock2.is_acquired() is True
lock2.release()
lock3 = etcd.lock('lock-5', ttl=2)
lock3.acquire()
assert lock3.is_acquired() is True
assert lock2.is_acquired() is False
def test_lock_context_manager(self, etcd):
with etcd.lock('lock-6', ttl=2) as lock:
assert lock.is_acquired() is True
assert lock.is_acquired() is False
def test_lock_contended(self, etcd):
lock1 = etcd.lock('lock-7', ttl=2)
lock1.acquire()
lock2 = etcd.lock('lock-7', ttl=2)
lock2.acquire()
assert lock1.is_acquired() is False
assert lock2.is_acquired() is True
def test_lock_double_acquire_release(self, etcd):
lock = etcd.lock('lock-8', ttl=10)
assert lock.acquire(0) is True
assert lock.acquire(0) is False
assert lock.release() is True
def test_lock_acquire_none(self, etcd):
lock = etcd.lock('lock-9', ttl=10)
assert lock.acquire(None) is True
# This will succeed after 10 seconds since the TTL will expire and the
# lock is not refreshed
assert lock.acquire(None) is True
def test_internal_exception_on_internal_error(self, etcd):
exception = self.MockedException(grpc.StatusCode.INTERNAL)
kv_mock = mock.MagicMock()
kv_mock.Range.side_effect = exception
etcd.kvstub = kv_mock
with pytest.raises(etcd3.exceptions.InternalServerError):
etcd.get("foo")
def test_connection_failure_exception_on_connection_failure(self, etcd):
exception = self.MockedException(grpc.StatusCode.UNAVAILABLE)
kv_mock = mock.MagicMock()
kv_mock.Range.side_effect = exception
etcd.kvstub = kv_mock
with pytest.raises(etcd3.exceptions.ConnectionFailedError):
etcd.get("foo")
def test_connection_timeout_exception_on_connection_timeout(self, etcd):
exception = self.MockedException(grpc.StatusCode.DEADLINE_EXCEEDED)
kv_mock = mock.MagicMock()
kv_mock.Range.side_effect = exception
etcd.kvstub = kv_mock
with pytest.raises(etcd3.exceptions.ConnectionTimeoutError):
etcd.get("foo")
def test_grpc_exception_on_unknown_code(self, etcd):
exception = self.MockedException(grpc.StatusCode.DATA_LOSS)
kv_mock = mock.MagicMock()
kv_mock.Range.side_effect = exception
etcd.kvstub = kv_mock
with pytest.raises(grpc.RpcError):
etcd.get("foo")
def test_status_member(self, etcd):
status = etcd.status()
assert isinstance(status.leader, etcd3.members.Member) is True
assert status.leader.id in [m.id for m in etcd.members]
def test_hash(self, etcd):
assert isinstance(etcd.hash(), int)
def test_snapshot(self, etcd):
with tempfile.NamedTemporaryFile() as f:
etcd.snapshot(f)
f.flush()
etcdctl('snapshot', 'status', f.name)
def test_find_something_rare():
st = characters(whitelist_categories=["Zs"], min_codepoint=12288)
find_any(st, lambda c: unicodedata.category(c) == "Zs")
assert_no_examples(st, lambda c: unicodedata.category(c) != "Zs")
from stat import S_IRUSR, S_IWUSR
from os.path import abspath
import pytest
from hypothesis import given, example
import hypothesis.strategies as strat
from vault_anyconfig.vault_anyconfig import VaultAnyConfig
@patch("vault_anyconfig.vault_anyconfig.chmod")
@patch("vault_anyconfig.vault_anyconfig.dump_base")
@patch("vault_anyconfig.vault_anyconfig.Client.read")
@given(
file_path=strat.text(
min_size=1, alphabet=strat.characters(blacklist_categories=("C"))),
secret_path=strat.text(
min_size=1,
alphabet=strat.characters(
blacklist_categories=("C"),
blacklist_characters=
".", # Since we separately specify the key, we cannot include "." in the secret path
),
),
secret_key=strat.text(
min_size=1,
alphabet=strat.characters(
blacklist_categories=("C"),
blacklist_characters=
".", # Keys require actual values, not more dot separators
),
