def from_file(
infile,
*,
_read_tsv=read_tsv,
):
"""Return the info for known declarations in the given file."""
known = {
'variables': {},
#'types': {},
#'constants': {},
#'macros': {},
}
for row in _read_tsv(infile, HEADER):
filename, funcname, name, kind, declaration = row
if not funcname or funcname == '-':
funcname = None
id = ID(filename, funcname, name)
if kind == 'variable':
values = known['variables']
value = Variable(id, declaration)
value._isglobal = _is_global(declaration) or id.funcname is None
else:
raise ValueError(f'unsupported kind in row {row}')
if value.name == 'id' and declaration == UNKNOWN:
# None of these are variables.
declaration = 'int id'
else:
value.validate()
values[id] = value
return known
def test_validate_bad_field(self):
badch = tuple(c for c in string.punctuation + string.digits)
notnames = (
'1a',
'a.b',
'a-b',
'&a',
'a++',
) + badch
tests = [
('id', ()), # Any non-empty str is okay.
('vartype', ()), # Any non-empty str is okay.
]
seen = set()
for field, invalid in tests:
for value in invalid:
seen.add(value)
with self.subTest(f'{field}={value!r}'):
static = Variable(**self.VALID_KWARGS)
static = static._replace(**{field: value})
with self.assertRaises(ValueError):
static.validate()
for field, invalid in tests:
valid = seen - set(invalid)
for value in valid:
with self.subTest(f'{field}={value!r}'):
static = Variable(**self.VALID_KWARGS)
static = static._replace(**{field: value})
static.validate() # This does not fail.
def from_file(
infile,
*,
_read_tsv=read_tsv,
):
"""Return the info for known declarations in the given file."""
known = {
'variables': {},
#'types': {},
#'constants': {},
#'macros': {},
}
for row in _read_tsv(infile, HEADER):
filename, funcname, name, kind, declaration = row
if not funcname or funcname == '-':
funcname = None
id = ID(filename, funcname, name)
if kind == 'variable':
values = known['variables']
if funcname:
storage = _get_storage(declaration) or 'local'
else:
storage = _get_storage(declaration) or 'implicit'
value = Variable(id, storage, declaration)
else:
raise ValueError(f'unsupported kind in row {row}')
value.validate()
# if value.name == 'id' and declaration == UNKNOWN:
# # None of these are variables.
# declaration = 'int id';
# else:
# value.validate()
values[id] = value
return known
def test_validate_missing_field(self):
for field in Variable._fields:
with self.subTest(field):
static = Variable(**self.VALID_KWARGS)
static = static._replace(**{field: None})
with self.assertRaises(TypeError):
static.validate()
def test_validate_typical(self):
static = Variable(
id=ID(
filename='x/y/z/spam.c',
funcname='func',
name='eggs',
),
vartype='int',
)
static.validate() # This does not fail.
def known_rows(
symbols,
*,
cached=True,
_get_filenames=iter_cpython_files,
_find_match=find_matching_variable,
_find_symbols=find_variables,
_as_known=known_row,
):
filenames = list(_get_filenames())
cache = {}
if cached:
for symbol in symbols:
try:
found = _known(symbol)
except KeyError:
found = _find_match(symbol, cache, filenames)
if found is None:
found = Variable(symbol.id, UNKNOWN, UNKNOWN)
yield _as_known(found.id, found.vartype)
else:
raise NotImplementedError # XXX incorporate KNOWN
for variable in _find_symbols(
symbols,
filenames,
srccache=cache,
parse_variable=_parse_global,
):
#variable = variable._replace(
# filename=os.path.relpath(variable.filename, REPO_ROOT))
if variable.funcname == UNKNOWN:
print(variable)
if variable.vartype == UNKNOWN:
print(variable)
yield _as_known(variable.id, variable.vartype)
def test_iterable(self):
static = Variable(**self.VALID_KWARGS)
id, vartype = static
values = (id, vartype)
for value, expected in zip(values, self.VALID_EXPECTED):
self.assertEqual(value, expected)
def test_init_all_missing(self):
for value in ('', None):
with self.subTest(repr(value)):
static = Variable(
id=value,
vartype=value,
)
self.assertEqual(static, (
None,
None,
))
def test_init_typical_local(self):
static = Variable(
id=ID(
filename='x/y/z/spam.c',
funcname='func',
name='eggs',
),
vartype='int',
)
self.assertEqual(static, (
('x/y/z/spam.c', 'func', 'eggs'),
'int',
))
def test_typical(self):
lines = textwrap.dedent('''
filename funcname name kind declaration
file1.c - var1 variable static int
file1.c func1 local1 variable static int
file1.c - var2 variable int
file1.c func2 local2 variable char *
file2.c - var1 variable char *
''').strip().splitlines()
lines = [re.sub(r'\s+', '\t', line, 4) for line in lines]
self._return_read_tsv = [
tuple(v.strip() for v in line.split('\t')) for line in lines[1:]
]
known = from_file('spam.c', _read_tsv=self._read_tsv)
self.assertEqual(
known, {
'variables': {
v.id: v
for v in [
Variable.from_parts('file1.c', '', 'var1',
'static int'),
Variable.from_parts('file1.c', 'func1', 'local1',
'static int'),
Variable.from_parts('file1.c', '', 'var2', 'int'),
Variable.from_parts('file1.c', 'func2', 'local2',
'char *'),
Variable.from_parts('file2.c', '', 'var1', 'char *'),
]
},
})
self.assertEqual(self.calls, [
('_read_tsv',
('spam.c', 'filename\tfuncname\tname\tkind\tdeclaration')),
])
def test_init_all_coerced(self):
id = ID('x/y/z/spam.c', 'func', 'spam')
tests = [
('str subclass',
dict(
id=(
PseudoStr('x/y/z/spam.c'),
PseudoStr('func'),
PseudoStr('spam'),
),
vartype=PseudoStr('int'),
), (
id,
'int',
)),
('non-str 1', dict(
id=id,
vartype=Object(),
), (
id,
'<object>',
)),
('non-str 2', dict(
id=id,
vartype=StrProxy('variable'),
), (
id,
'variable',
)),
('non-str', dict(
id=id,
vartype=('a', 'b', 'c'),
), (
id,
"('a', 'b', 'c')",
)),
]
for summary, kwargs, expected in tests:
with self.subTest(summary):
static = Variable(**kwargs)
for field in Variable._fields:
value = getattr(static, field)
if field == 'id':
self.assertIs(type(value), ID)
else:
self.assertIs(type(value), str)
self.assertEqual(tuple(static), expected)
def _known(symbol):
if symbol.funcname:
if symbol.funcname != UNKNOWN or symbol.filename != UNKNOWN:
raise KeyError(symbol.name)
filename, funcname, decl = LOCALS[symbol.name]
varid = ID(filename, funcname, symbol.name)
elif not symbol.filename or symbol.filename == UNKNOWN:
raise KeyError(symbol.name)
else:
varid = symbol.id
try:
decl = GLOBALS[symbol.name]
except KeyError:
if symbol.name.endswith('_methods'):
decl = 'static PyMethodDef '
elif symbol.filename == 'Objects/exceptions.c' and symbol.name.startswith(('PyExc_', '_PyExc_')):
decl = 'static PyTypeObject '
else:
raise
if symbol.name not in decl:
decl = decl + symbol.name
return Variable(varid, decl)
def test___getattr__(self):
static = Variable(('a', 'b', 'z'), 'x')
self.assertEqual(static.filename, 'a')
self.assertEqual(static.funcname, 'b')
self.assertEqual(static.name, 'z')
def test_fields(self):
static = Variable(('a', 'b', 'z'), 'x')
self.assertEqual(static.id, ('a', 'b', 'z'))
self.assertEqual(static.vartype, 'x')