def collect_types_fixture():
if DUMP_PYANNOTATE:
from pyannotate_runtime import collect_types
collect_types.start()
yield
if DUMP_PYANNOTATE:
collect_types.stop()
def collecting_types(self):
# type: () -> Iterator[None]
collect_types.collected_args = {}
collect_types.collected_signatures = {}
collect_types.num_samples = {}
collect_types.sampling_counters = {}
collect_types.call_pending = set()
collect_types.start()
yield None
collect_types.stop()
self.load_stats()
def test_only_return(self):
# type: () -> None
def only_return(x):
# type: (int) -> str
collect_types.start()
return ''
only_return(1)
collect_types.stop()
self.load_stats()
# No entry is stored if we only have a return event with no matching call.
self.assert_type_comments('only_return', [])
def annotations_collect():
"""Fixture to collect 'annotations' for pyannonate
Yields:
[None]: [Generates file annotations.txt]
"""
# pipx install pyannotate==1.2.0
from pyannotate_runtime import collect_types
collect_types.init_types_collection()
collect_types.start()
yield None
collect_types.stop()
collect_types.dump_stats("annotations.txt")
def main():
argv = sys.argv[1:]
ignore_globals = list(six.moves.builtins.__dict__)
# Parse arguments
FORMAT = Format.ANSI
OUTPUTFILE = sys.stdout
LOCAL = 0
NUM = None
KW_OK = []
KW_KO = []
DUMP = 0
CRC = True
BREAKFAILED = True
ONLYFAILED = False
VERB = 3
GLOB_PREEXEC = ""
PREEXEC_DICT = {}
SCAPY = "scapy"
MODULES = []
TESTFILES = []
ANNOTATIONS_MODE = False
try:
opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DdCFqP:s:x")
for opt, optarg in opts[0]:
if opt == "-h":
usage()
elif opt == "-b":
BREAKFAILED = False
elif opt == "-F":
ONLYFAILED = True
elif opt == "-q":
VERB -= 1
elif opt == "-D":
DUMP = 2
elif opt == "-d":
DUMP = 1
elif opt == "-C":
CRC = False
elif opt == "-x":
ANNOTATIONS_MODE = True
elif opt == "-s":
SCAPY = optarg
elif opt == "-P":
GLOB_PREEXEC += "\n" + optarg
elif opt == "-f":
try:
FORMAT = Format.from_string(optarg)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
elif opt == "-t":
TESTFILES.append(optarg)
TESTFILES = resolve_testfiles(TESTFILES)
elif opt == "-T":
TESTFILES.remove(optarg)
elif opt == "-c":
data = parse_config_file(optarg, VERB)
BREAKFAILED = data.breakfailed
ONLYFAILED = data.onlyfailed
VERB = data.verb
DUMP = data.dump
CRC = data.crc
SCAPY = data.scapy
PREEXEC_DICT = data.preexec
GLOB_PREEXEC = data.global_preexec
OUTPUTFILE = data.outfile
TESTFILES = data.testfiles
LOCAL = 1 if data.local else 0
NUM = data.num
MODULES = data.modules
KW_OK.extend(data.kw_ok)
KW_KO.extend(data.kw_ko)
try:
FORMAT = Format.from_string(data.format)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
TESTFILES = resolve_testfiles(TESTFILES)
for testfile in resolve_testfiles(data.remove_testfiles):
try:
TESTFILES.remove(testfile)
except ValueError:
error_m = "Cannot remove %s from test files" % testfile
raise getopt.GetoptError(error_m)
elif opt == "-o":
OUTPUTFILE = optarg
if not os.access(os.path.dirname(os.path.abspath(OUTPUTFILE)), os.W_OK):
raise getopt.GetoptError("Cannot write to file %s" % OUTPUTFILE)
elif opt == "-l":
LOCAL = 1
elif opt == "-n":
NUM = []
for v in (x.strip() for x in optarg.split(",")):
try:
NUM.append(int(v))
except ValueError:
v1, v2 = [int(e) for e in v.split('-', 1)]
NUM.extend(range(v1, v2 + 1))
elif opt == "-m":
MODULES.append(optarg)
elif opt == "-k":
KW_OK.extend(optarg.split(","))
elif opt == "-K":
KW_KO.extend(optarg.split(","))
# Discard Python3 tests when using Python2
if six.PY2:
KW_KO.append("python3_only")
if ANNOTATIONS_MODE:
try:
from pyannotate_runtime import collect_types
except ImportError:
raise ImportError("Please install pyannotate !")
collect_types.init_types_collection()
collect_types.start()
if VERB > 2:
print("### Booting scapy...", file=sys.stderr)
try:
from scapy import all as scapy
except Exception as e:
print("[CRITICAL]: Cannot import Scapy: %s" % e, file=sys.stderr)
traceback.print_exc()
sys.exit(1) # Abort the tests
for m in MODULES:
try:
mod = import_module(m)
six.moves.builtins.__dict__.update(mod.__dict__)
except ImportError as e:
raise getopt.GetoptError("cannot import [%s]: %s" % (m, e))
if WINDOWS:
from scapy.arch.windows import route_add_loopback
route_add_loopback()
# Add SCAPY_ROOT_DIR environment variable, used for tests
os.environ['SCAPY_ROOT_DIR'] = os.environ.get("PWD", os.getcwd())
except getopt.GetoptError as msg:
print("ERROR:", msg, file=sys.stderr)
raise SystemExit
autorun_func = {
Format.TEXT: scapy.autorun_get_text_interactive_session,
Format.ANSI: scapy.autorun_get_ansi_interactive_session,
Format.HTML: scapy.autorun_get_html_interactive_session,
Format.LATEX: scapy.autorun_get_latex_interactive_session,
Format.XUNIT: scapy.autorun_get_text_interactive_session,
}
if VERB > 2:
print("### Starting tests...", file=sys.stderr)
glob_output = ""
glob_result = 0
glob_title = None
UNIQUE = len(TESTFILES) == 1
# Resolve tags and asterix
for prex in six.iterkeys(copy.copy(PREEXEC_DICT)):
if "*" in prex:
pycode = PREEXEC_DICT[prex]
del PREEXEC_DICT[prex]
for gl in glob.iglob(prex):
_pycode = pycode.replace("%name%", os.path.splitext(os.path.split(gl)[1])[0]) # noqa: E501
PREEXEC_DICT[gl] = _pycode
pos_begin = 0
runned_campaigns = []
# Execute all files
for TESTFILE in TESTFILES:
if VERB > 2:
print("### Loading:", TESTFILE, file=sys.stderr)
PREEXEC = PREEXEC_DICT[TESTFILE] if TESTFILE in PREEXEC_DICT else GLOB_PREEXEC
with open(TESTFILE) as testfile:
output, result, campaign = execute_campaign(testfile, OUTPUTFILE,
PREEXEC, NUM, KW_OK, KW_KO,
DUMP, FORMAT, VERB, ONLYFAILED,
CRC, autorun_func, pos_begin,
ignore_globals)
runned_campaigns.append(campaign)
pos_begin = campaign.end_pos
if UNIQUE:
glob_title = campaign.title
glob_output += output
if not result:
glob_result = 1
if BREAKFAILED:
break
if VERB > 2:
print("### Writing output...", file=sys.stderr)
if ANNOTATIONS_MODE:
collect_types.stop()
collect_types.dump_stats("pyannotate_results")
# Concenate outputs
if FORMAT == Format.HTML:
glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL, glob_title)
# Write the final output
# Note: on Python 2, we force-encode to ignore ascii errors
# on Python 3, we need to detect the type of stream
if OUTPUTFILE == sys.stdout:
OUTPUTFILE.write(glob_output.encode("utf8", "ignore")
if 'b' in OUTPUTFILE.mode or six.PY2 else glob_output)
else:
with open(OUTPUTFILE, "wb") as f:
f.write(glob_output.encode("utf8", "ignore")
if 'b' in f.mode or six.PY2 else glob_output)
# Delete scapy's test environment vars
del os.environ['SCAPY_ROOT_DIR']
# Return state
return glob_result
def main():
argv = sys.argv[1:]
logger = logging.getLogger("scapy")
logger.addHandler(logging.StreamHandler())
import scapy
print(dash + " UTScapy - Scapy %s - %s" %
(scapy.__version__, sys.version.split(" ")[0]))
# Parse arguments
FORMAT = Format.ANSI
OUTPUTFILE = sys.stdout
LOCAL = 0
NUM = None
NON_ROOT = False
KW_OK = []
KW_KO = []
DUMP = 0
DOCS = 0
CRC = True
BREAKFAILED = True
ONLYFAILED = False
VERB = 3
GLOB_PREEXEC = ""
PREEXEC_DICT = {}
MODULES = []
TESTFILES = []
ANNOTATIONS_MODE = False
INTERPRETER = False
try:
opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DRdCiFqNP:s:x")
for opt, optarg in opts[0]:
if opt == "-h":
usage()
elif opt == "-b":
BREAKFAILED = False
elif opt == "-F":
ONLYFAILED = True
elif opt == "-q":
VERB -= 1
elif opt == "-D":
DUMP = 2
elif opt == "-R":
DOCS = 1
elif opt == "-d":
DUMP = 1
elif opt == "-C":
CRC = False
elif opt == "-i":
INTERPRETER = True
elif opt == "-x":
ANNOTATIONS_MODE = True
elif opt == "-P":
GLOB_PREEXEC += "\n" + optarg
elif opt == "-f":
try:
FORMAT = Format.from_string(optarg)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
elif opt == "-t":
TESTFILES.append(optarg)
TESTFILES = resolve_testfiles(TESTFILES)
elif opt == "-T":
TESTFILES.remove(optarg)
elif opt == "-c":
data = parse_config_file(optarg, VERB)
BREAKFAILED = data.breakfailed
ONLYFAILED = data.onlyfailed
VERB = data.verb
DUMP = data.dump
CRC = data.crc
PREEXEC_DICT = data.preexec
GLOB_PREEXEC = data.global_preexec
OUTPUTFILE = data.outfile
TESTFILES = data.testfiles
LOCAL = 1 if data.local else 0
NUM = data.num
MODULES = data.modules
KW_OK.extend(data.kw_ok)
KW_KO.extend(data.kw_ko)
try:
FORMAT = Format.from_string(data.format)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
TESTFILES = resolve_testfiles(TESTFILES)
for testfile in resolve_testfiles(data.remove_testfiles):
try:
TESTFILES.remove(testfile)
except ValueError:
error_m = "Cannot remove %s from test files" % testfile
raise getopt.GetoptError(error_m)
elif opt == "-o":
OUTPUTFILE = optarg
if not os.access(os.path.dirname(os.path.abspath(OUTPUTFILE)),
os.W_OK):
raise getopt.GetoptError("Cannot write to file %s" %
OUTPUTFILE)
elif opt == "-l":
LOCAL = 1
elif opt == "-n":
NUM = []
for v in (x.strip() for x in optarg.split(",")):
try:
NUM.append(int(v))
except ValueError:
v1, v2 = [int(e) for e in v.split('-', 1)]
NUM.extend(range(v1, v2 + 1))
elif opt == "-N":
NON_ROOT = True
elif opt == "-m":
MODULES.append(optarg)
elif opt == "-k":
KW_OK.extend(optarg.split(","))
elif opt == "-K":
KW_KO.extend(optarg.split(","))
except getopt.GetoptError as msg:
print("ERROR:", msg)
raise SystemExit
if FORMAT in [Format.LIVE, Format.ANSI]:
theme = DefaultTheme()
else:
theme = BlackAndWhite()
# Disable tests if needed
# Discard Python3 tests when using Python2
if six.PY2:
KW_KO.append("python3_only")
if VERB > 2:
print(" " + arrow + " Python 2 mode")
try:
if NON_ROOT or os.getuid() != 0: # Non root
# Discard root tests
KW_KO.append("needs_root")
if VERB > 2:
print(" " + arrow + " Non-root mode")
except AttributeError:
pass
if conf.use_pcap or WINDOWS:
KW_KO.append("not_libpcap")
if VERB > 2:
print(" " + arrow + " libpcap mode")
KW_KO.append("disabled")
# Process extras
if six.PY3:
KW_KO.append("FIXME_py3")
if ANNOTATIONS_MODE:
try:
from pyannotate_runtime import collect_types
except ImportError:
raise ImportError("Please install pyannotate !")
collect_types.init_types_collection()
collect_types.start()
if VERB > 2:
print(" " + arrow + " Booting scapy...")
try:
from scapy import all as scapy
except Exception as e:
print("[CRITICAL]: Cannot import Scapy: %s" % e)
traceback.print_exc()
sys.exit(1) # Abort the tests
for m in MODULES:
try:
mod = import_module(m)
six.moves.builtins.__dict__.update(mod.__dict__)
except ImportError as e:
raise getopt.GetoptError("cannot import [%s]: %s" % (m, e))
autorun_func = {
Format.TEXT: scapy.autorun_get_text_interactive_session,
Format.ANSI: scapy.autorun_get_ansi_interactive_session,
Format.HTML: scapy.autorun_get_html_interactive_session,
Format.LATEX: scapy.autorun_get_latex_interactive_session,
Format.XUNIT: scapy.autorun_get_text_interactive_session,
Format.LIVE: scapy.autorun_get_live_interactive_session,
}
if VERB > 2:
print(" " + arrow + " Discovering tests files...")
glob_output = ""
glob_result = 0
glob_title = None
UNIQUE = len(TESTFILES) == 1
# Resolve tags and asterix
for prex in six.iterkeys(copy.copy(PREEXEC_DICT)):
if "*" in prex:
pycode = PREEXEC_DICT[prex]
del PREEXEC_DICT[prex]
for gl in glob.iglob(prex):
_pycode = pycode.replace(
"%name%",
os.path.splitext(os.path.split(gl)[1])[0]) # noqa: E501
PREEXEC_DICT[gl] = _pycode
pos_begin = 0
runned_campaigns = []
from scapy.main import _scapy_builtins
scapy_ses = _scapy_builtins()
import_UTscapy_tools(scapy_ses)
# Execute all files
for TESTFILE in TESTFILES:
if VERB > 2:
print(theme.green(dash + " Loading: %s" % TESTFILE))
PREEXEC = PREEXEC_DICT[
TESTFILE] if TESTFILE in PREEXEC_DICT else GLOB_PREEXEC
with open(TESTFILE) as testfile:
output, result, campaign = execute_campaign(
testfile,
OUTPUTFILE,
PREEXEC,
NUM,
KW_OK,
KW_KO,
DUMP,
DOCS,
FORMAT,
VERB,
ONLYFAILED,
CRC,
INTERPRETER,
autorun_func,
theme,
pos_begin=pos_begin,
scapy_ses=copy.copy(scapy_ses))
runned_campaigns.append(campaign)
pos_begin = campaign.end_pos
if UNIQUE:
glob_title = campaign.title
glob_output += output
if not result:
glob_result = 1
if BREAKFAILED:
break
if VERB > 2:
print(checkmark + " All campaigns executed. Writing output...")
if ANNOTATIONS_MODE:
collect_types.stop()
collect_types.dump_stats("pyannotate_results")
# Concenate outputs
if FORMAT == Format.HTML:
glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL,
glob_title)
if FORMAT == Format.LATEX:
glob_output = pack_latex_campaigns(runned_campaigns, glob_output,
LOCAL, glob_title)
# Write the final output
# Note: on Python 2, we force-encode to ignore ascii errors
# on Python 3, we need to detect the type of stream
if OUTPUTFILE == sys.stdout:
print(glob_output, file=OUTPUTFILE)
else:
with open(OUTPUTFILE, "wb") as f:
f.write(
glob_output.encode("utf8", "ignore")
if 'b' in f.mode or six.PY2 else glob_output)
# Print end message
if VERB > 2:
if glob_result == 0:
print(theme.green("UTscapy ended successfully"))
else:
print(theme.red("UTscapy ended with error code %s" % glob_result))
# Check active threads
if VERB > 2:
if threading.active_count() > 1:
print("\nWARNING: UNFINISHED THREADS")
print(threading.enumerate())
import multiprocessing
processes = multiprocessing.active_children()
if processes:
print("\nWARNING: UNFINISHED PROCESSES")
print(processes)
sys.stdout.flush()
# Return state
return glob_result
def collect_types_fixture():
collect_types.start()
yield
collect_types.stop()
def collect_types_fixture():
from pyannotate_runtime import collect_types
collect_types.start()
yield
collect_types.stop()
