def pytest_sessionfinish(session):
plugin_base.final_process_cleanup()
if session.config.option.dump_pyannotate:
from pyannotate_runtime import collect_types
collect_types.dump_stats(session.config.option.dump_pyannotate)
def main():
collect_types.init_types_collection()
with collect_types.collect():
os.chdir('/home/maciej/Desktop/REPO')
cmd.launch(['status'])
cmd.launch(['file'])
cmd.launch(['format'])
cmd.launch(['help'])
os.system('rm -f .git/machete')
cmd.launch(['add', '--yes'])
cmd.launch(['anno'])
cmd.launch(['discover', '--yes'])
# # cmd.launch(['e'])
cmd.launch(['delete-unmanaged'])
cmd.launch(['g','f'])
cmd.launch(['list','managed'])
cmd.launch(['show','f'])
# # # cmd.launch(['d'])
cmd.launch(['fork-point'])
# # # cmd.launch(['l'])
cmd.launch(['traverse'])
# # # cmd.launch(['update'])
os.chdir(file_dir)
collect_types.dump_stats(json_name)
if DEBUG == True:
if os.path.exists(json_name):
print('dziala!!)
# del_json()
else:
print(':C')
else:
create_types()
print('Fajno')
def main():
loader = unittest.TestLoader()
g = glob.glob
py = g('frontend/*_test.py') + g('osh/*_test.py') + g(
'core/*_test.py') + g('')
# hangs
py.remove('core/process_test.py')
modules = []
for p in py:
mod_name = p[:-3].replace('/', '.')
print(mod_name)
modules.append(__import__(mod_name, fromlist=['.']))
for m in modules:
print(m)
suites = [loader.loadTestsFromModule(m) for m in modules]
suite = unittest.TestSuite()
for s in suites:
suite.addTest(s)
runner = unittest.TextTestRunner()
collect_types.init_types_collection()
with collect_types.collect():
runner.run(suite)
if 0:
TopLevel()
Match()
Arith()
collect_types.dump_stats('type_info.json')
def main_pyannotate():
try:
collect_types.init_types_collection()
collect_types.resume()
return main()
finally:
log.info("dumping type_info")
collect_types.pause()
collect_types.dump_stats('type_info.json')
def main():
import sys
from pyannotate_runtime import collect_types
collect_types.init_types_collection()
with collect_types.collect():
dog = Dog('sharik')
dog.name
dog.bark('ru')
collect_types.dump_stats('type_info.json')
def pytest_sessionfinish(session):
from sqlalchemy.testing import asyncio
asyncio._maybe_async_provisioning(plugin_base.final_process_cleanup)
if session.config.option.dump_pyannotate:
from pyannotate_runtime import collect_types
collect_types.dump_stats(session.config.option.dump_pyannotate)
def main_annotate():
from pyannotate_runtime import collect_types
collect_types.init_types_collection(filter_filename=filter_filename)
collect_types.resume()
try:
manager.run()
except BaseException:
pass
collect_types.dump_stats(b"type_info.json")
def annotations_collect():
"""Fixture to collect 'annotations' for pyannonate
Yields:
[None]: [Generates file annotations.txt]
"""
# pipx install pyannotate==1.2.0
from pyannotate_runtime import collect_types
collect_types.init_types_collection()
collect_types.start()
yield None
collect_types.stop()
collect_types.dump_stats("annotations.txt")
def test_input(self, capsys):
collect_types.init_types_collection()
with collect_types.collect():
assert len(sqalp.known_formats) > 1
assert sqalp.get_session(logging.ERROR)
session = sqalp.get_session(logging.DEBUG)
parser = sqalp.get_parser(sqalp.known_formats['common'])
sqalp.file_import(session, open(f'{temp}/sample.log', 'r'), parser,
False)
results = sqalp.get_by_date(session)
assert results != TestClass.count_json
results = sqalp.get_by_date_by_ua(session)
assert results != TestClass.ua_json
results = sqalp.get_by_date_verb_ratio(session)
assert results != TestClass.ration_json
output_data = OrderedDict([('2011-12-01', [2822]),
('2011-12-02', [2572]),
('2011-12-03', [604])])
output_print_data_default = \
"+--------------+--------------+--------------+\n" \
"| 2011-12-01 | 2011-12-02 | 2011-12-03 |\n" \
"+==============+==============+==============+\n" \
"| 2822 | 2572 | 604 |\n" \
"+--------------+--------------+--------------+\n"
output_print_data_json = \
'{"2011-12-01": [2822], ' \
'"2011-12-02": [2572], ' \
'"2011-12-03": [604]}'
sqalp.output(output_data, 'grid')
captured = capsys.readouterr()
assert output_print_data_default in captured.out
sqalp.output(output_data, 'json')
captured = capsys.readouterr()
assert output_print_data_json in captured.out
sqalp.setup_logging(logging.ERROR)
# Write collectd type data so it can be imported into source
# Not in PWD ot tox/setup get miffed
collect_types.dump_stats('/tmp/types.py')
def main():
argv = sys.argv[1:]
logger = logging.getLogger("scapy")
logger.addHandler(logging.StreamHandler())
import scapy
print(dash + " UTScapy - Scapy %s - %s" %
(scapy.__version__, sys.version.split(" ")[0]))
# Parse arguments
FORMAT = Format.ANSI
OUTPUTFILE = sys.stdout
LOCAL = 0
NUM = None
NON_ROOT = False
KW_OK = []
KW_KO = []
DUMP = 0
DOCS = 0
CRC = True
BREAKFAILED = True
ONLYFAILED = False
VERB = 3
GLOB_PREEXEC = ""
PREEXEC_DICT = {}
MODULES = []
TESTFILES = []
ANNOTATIONS_MODE = False
INTERPRETER = False
try:
opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DRdCiFqNP:s:x")
for opt, optarg in opts[0]:
if opt == "-h":
usage()
elif opt == "-b":
BREAKFAILED = False
elif opt == "-F":
ONLYFAILED = True
elif opt == "-q":
VERB -= 1
elif opt == "-D":
DUMP = 2
elif opt == "-R":
DOCS = 1
elif opt == "-d":
DUMP = 1
elif opt == "-C":
CRC = False
elif opt == "-i":
INTERPRETER = True
elif opt == "-x":
ANNOTATIONS_MODE = True
elif opt == "-P":
GLOB_PREEXEC += "\n" + optarg
elif opt == "-f":
try:
FORMAT = Format.from_string(optarg)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
elif opt == "-t":
TESTFILES.append(optarg)
TESTFILES = resolve_testfiles(TESTFILES)
elif opt == "-T":
TESTFILES.remove(optarg)
elif opt == "-c":
data = parse_config_file(optarg, VERB)
BREAKFAILED = data.breakfailed
ONLYFAILED = data.onlyfailed
VERB = data.verb
DUMP = data.dump
CRC = data.crc
PREEXEC_DICT = data.preexec
GLOB_PREEXEC = data.global_preexec
OUTPUTFILE = data.outfile
TESTFILES = data.testfiles
LOCAL = 1 if data.local else 0
NUM = data.num
MODULES = data.modules
KW_OK.extend(data.kw_ok)
KW_KO.extend(data.kw_ko)
try:
FORMAT = Format.from_string(data.format)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
TESTFILES = resolve_testfiles(TESTFILES)
for testfile in resolve_testfiles(data.remove_testfiles):
try:
TESTFILES.remove(testfile)
except ValueError:
error_m = "Cannot remove %s from test files" % testfile
raise getopt.GetoptError(error_m)
elif opt == "-o":
OUTPUTFILE = optarg
if not os.access(os.path.dirname(os.path.abspath(OUTPUTFILE)),
os.W_OK):
raise getopt.GetoptError("Cannot write to file %s" %
OUTPUTFILE)
elif opt == "-l":
LOCAL = 1
elif opt == "-n":
NUM = []
for v in (x.strip() for x in optarg.split(",")):
try:
NUM.append(int(v))
except ValueError:
v1, v2 = [int(e) for e in v.split('-', 1)]
NUM.extend(range(v1, v2 + 1))
elif opt == "-N":
NON_ROOT = True
elif opt == "-m":
MODULES.append(optarg)
elif opt == "-k":
KW_OK.extend(optarg.split(","))
elif opt == "-K":
KW_KO.extend(optarg.split(","))
except getopt.GetoptError as msg:
print("ERROR:", msg)
raise SystemExit
if FORMAT in [Format.LIVE, Format.ANSI]:
theme = DefaultTheme()
else:
theme = BlackAndWhite()
# Disable tests if needed
# Discard Python3 tests when using Python2
if six.PY2:
KW_KO.append("python3_only")
if VERB > 2:
print(" " + arrow + " Python 2 mode")
try:
if NON_ROOT or os.getuid() != 0: # Non root
# Discard root tests
KW_KO.append("needs_root")
if VERB > 2:
print(" " + arrow + " Non-root mode")
except AttributeError:
pass
if conf.use_pcap or WINDOWS:
KW_KO.append("not_libpcap")
if VERB > 2:
print(" " + arrow + " libpcap mode")
KW_KO.append("disabled")
# Process extras
if six.PY3:
KW_KO.append("FIXME_py3")
if ANNOTATIONS_MODE:
try:
from pyannotate_runtime import collect_types
except ImportError:
raise ImportError("Please install pyannotate !")
collect_types.init_types_collection()
collect_types.start()
if VERB > 2:
print(" " + arrow + " Booting scapy...")
try:
from scapy import all as scapy
except Exception as e:
print("[CRITICAL]: Cannot import Scapy: %s" % e)
traceback.print_exc()
sys.exit(1) # Abort the tests
for m in MODULES:
try:
mod = import_module(m)
six.moves.builtins.__dict__.update(mod.__dict__)
except ImportError as e:
raise getopt.GetoptError("cannot import [%s]: %s" % (m, e))
autorun_func = {
Format.TEXT: scapy.autorun_get_text_interactive_session,
Format.ANSI: scapy.autorun_get_ansi_interactive_session,
Format.HTML: scapy.autorun_get_html_interactive_session,
Format.LATEX: scapy.autorun_get_latex_interactive_session,
Format.XUNIT: scapy.autorun_get_text_interactive_session,
Format.LIVE: scapy.autorun_get_live_interactive_session,
}
if VERB > 2:
print(" " + arrow + " Discovering tests files...")
glob_output = ""
glob_result = 0
glob_title = None
UNIQUE = len(TESTFILES) == 1
# Resolve tags and asterix
for prex in six.iterkeys(copy.copy(PREEXEC_DICT)):
if "*" in prex:
pycode = PREEXEC_DICT[prex]
del PREEXEC_DICT[prex]
for gl in glob.iglob(prex):
_pycode = pycode.replace(
"%name%",
os.path.splitext(os.path.split(gl)[1])[0]) # noqa: E501
PREEXEC_DICT[gl] = _pycode
pos_begin = 0
runned_campaigns = []
from scapy.main import _scapy_builtins
scapy_ses = _scapy_builtins()
import_UTscapy_tools(scapy_ses)
# Execute all files
for TESTFILE in TESTFILES:
if VERB > 2:
print(theme.green(dash + " Loading: %s" % TESTFILE))
PREEXEC = PREEXEC_DICT[
TESTFILE] if TESTFILE in PREEXEC_DICT else GLOB_PREEXEC
with open(TESTFILE) as testfile:
output, result, campaign = execute_campaign(
testfile,
OUTPUTFILE,
PREEXEC,
NUM,
KW_OK,
KW_KO,
DUMP,
DOCS,
FORMAT,
VERB,
ONLYFAILED,
CRC,
INTERPRETER,
autorun_func,
theme,
pos_begin=pos_begin,
scapy_ses=copy.copy(scapy_ses))
runned_campaigns.append(campaign)
pos_begin = campaign.end_pos
if UNIQUE:
glob_title = campaign.title
glob_output += output
if not result:
glob_result = 1
if BREAKFAILED:
break
if VERB > 2:
print(checkmark + " All campaigns executed. Writing output...")
if ANNOTATIONS_MODE:
collect_types.stop()
collect_types.dump_stats("pyannotate_results")
# Concenate outputs
if FORMAT == Format.HTML:
glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL,
glob_title)
if FORMAT == Format.LATEX:
glob_output = pack_latex_campaigns(runned_campaigns, glob_output,
LOCAL, glob_title)
# Write the final output
# Note: on Python 2, we force-encode to ignore ascii errors
# on Python 3, we need to detect the type of stream
if OUTPUTFILE == sys.stdout:
print(glob_output, file=OUTPUTFILE)
else:
with open(OUTPUTFILE, "wb") as f:
f.write(
glob_output.encode("utf8", "ignore")
if 'b' in f.mode or six.PY2 else glob_output)
# Print end message
if VERB > 2:
if glob_result == 0:
print(theme.green("UTscapy ended successfully"))
else:
print(theme.red("UTscapy ended with error code %s" % glob_result))
# Check active threads
if VERB > 2:
if threading.active_count() > 1:
print("\nWARNING: UNFINISHED THREADS")
print(threading.enumerate())
import multiprocessing
processes = multiprocessing.active_children()
if processes:
print("\nWARNING: UNFINISHED PROCESSES")
print(processes)
sys.stdout.flush()
# Return state
return glob_result
def pytest_sessionfinish(session, exitstatus):
os.makedirs('build/', exist_ok=True)
collect_types.dump_stats("build/type_info.json")
def report(self, stream):
collect_types.dump_stats('type_info.json')
import pytest
from pyannotate_runtime import collect_types
if __name__ == '__main__':
collect_types.init_types_collection()
with collect_types.collect():
pytest.main()
collect_types.dump_stats('type_info.json')
def pytest_sessionfinish(session, exitstatus):
collect_types.dump_stats("annotations.json")
# Called from Python-2.7.13/Modules/main.c.
def _cpython_main_hook():
sys.exit(main(sys.argv))
if __name__ == '__main__':
pyann_out = posix.environ.get('PYANN_OUT')
if pyann_out:
from pyannotate_runtime import collect_types
collect_types.init_types_collection()
with collect_types.collect():
status = main(sys.argv)
collect_types.dump_stats(pyann_out)
sys.exit(status)
elif posix.environ.get('RESOLVE') == '1':
from opy import resolve
resolve.Walk(dict(sys.modules))
elif posix.environ.get('CALLGRAPH') == '1':
# NOTE: This could end up as opy.InferTypes(), opy.GenerateCode(), etc.
from opy import callgraph
callgraph.Walk(main, sys.modules)
else:
sys.exit(main(sys.argv))
def pytest_sessionfinish(session, exitstatus):
from pyannotate_runtime import collect_types
collect_types.dump_stats("type_info.json")
def pytest_sessionfinish(session, exitstatus):
collect_types.dump_stats("type_info.json")
def pytest_unconfigure(self, config):
"""Unconfigure the pytest plugin. Happens when pytest is about to exit."""
collect_types.dump_stats(self.output_file)
from .patterns.constants import DataSetType
from .reader import download, process_data, save_data
num_cpus = os.cpu_count() or 4
# Make sure workers don't start a copy
ray.shutdown()
ray.init(num_cpus=num_cpus, include_dashboard=False)
def main():
# download()
for v in DataSetType:
data_set_type = v.value
print(f"[INFO] Processing {data_set_type} set")
processed = process_data(data_set_type, parallel=False)
save_data(processed, data_set_type)
if __name__ == "__main__":
# Collect runtime type data for pyannotate
collect_types.init_types_collection()
with collect_types.collect():
main()
collect_types.dump_stats("./pyannotate_runtime.stats")
def main():
argv = sys.argv[1:]
ignore_globals = list(six.moves.builtins.__dict__)
# Parse arguments
FORMAT = Format.ANSI
OUTPUTFILE = sys.stdout
LOCAL = 0
NUM = None
KW_OK = []
KW_KO = []
DUMP = 0
CRC = True
BREAKFAILED = True
ONLYFAILED = False
VERB = 3
GLOB_PREEXEC = ""
PREEXEC_DICT = {}
SCAPY = "scapy"
MODULES = []
TESTFILES = []
ANNOTATIONS_MODE = False
try:
opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DdCFqP:s:x")
for opt, optarg in opts[0]:
if opt == "-h":
usage()
elif opt == "-b":
BREAKFAILED = False
elif opt == "-F":
ONLYFAILED = True
elif opt == "-q":
VERB -= 1
elif opt == "-D":
DUMP = 2
elif opt == "-d":
DUMP = 1
elif opt == "-C":
CRC = False
elif opt == "-x":
ANNOTATIONS_MODE = True
elif opt == "-s":
SCAPY = optarg
elif opt == "-P":
GLOB_PREEXEC += "\n" + optarg
elif opt == "-f":
try:
FORMAT = Format.from_string(optarg)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
elif opt == "-t":
TESTFILES.append(optarg)
TESTFILES = resolve_testfiles(TESTFILES)
elif opt == "-T":
TESTFILES.remove(optarg)
elif opt == "-c":
data = parse_config_file(optarg, VERB)
BREAKFAILED = data.breakfailed
ONLYFAILED = data.onlyfailed
VERB = data.verb
DUMP = data.dump
CRC = data.crc
SCAPY = data.scapy
PREEXEC_DICT = data.preexec
GLOB_PREEXEC = data.global_preexec
OUTPUTFILE = data.outfile
TESTFILES = data.testfiles
LOCAL = 1 if data.local else 0
NUM = data.num
MODULES = data.modules
KW_OK.extend(data.kw_ok)
KW_KO.extend(data.kw_ko)
try:
FORMAT = Format.from_string(data.format)
except KeyError as msg:
raise getopt.GetoptError("Unknown output format %s" % msg)
TESTFILES = resolve_testfiles(TESTFILES)
for testfile in resolve_testfiles(data.remove_testfiles):
try:
TESTFILES.remove(testfile)
except ValueError:
error_m = "Cannot remove %s from test files" % testfile
raise getopt.GetoptError(error_m)
elif opt == "-o":
OUTPUTFILE = optarg
if not os.access(os.path.dirname(os.path.abspath(OUTPUTFILE)), os.W_OK):
raise getopt.GetoptError("Cannot write to file %s" % OUTPUTFILE)
elif opt == "-l":
LOCAL = 1
elif opt == "-n":
NUM = []
for v in (x.strip() for x in optarg.split(",")):
try:
NUM.append(int(v))
except ValueError:
v1, v2 = [int(e) for e in v.split('-', 1)]
NUM.extend(range(v1, v2 + 1))
elif opt == "-m":
MODULES.append(optarg)
elif opt == "-k":
KW_OK.extend(optarg.split(","))
elif opt == "-K":
KW_KO.extend(optarg.split(","))
# Discard Python3 tests when using Python2
if six.PY2:
KW_KO.append("python3_only")
if ANNOTATIONS_MODE:
try:
from pyannotate_runtime import collect_types
except ImportError:
raise ImportError("Please install pyannotate !")
collect_types.init_types_collection()
collect_types.start()
if VERB > 2:
print("### Booting scapy...", file=sys.stderr)
try:
from scapy import all as scapy
except Exception as e:
print("[CRITICAL]: Cannot import Scapy: %s" % e, file=sys.stderr)
traceback.print_exc()
sys.exit(1) # Abort the tests
for m in MODULES:
try:
mod = import_module(m)
six.moves.builtins.__dict__.update(mod.__dict__)
except ImportError as e:
raise getopt.GetoptError("cannot import [%s]: %s" % (m, e))
if WINDOWS:
from scapy.arch.windows import route_add_loopback
route_add_loopback()
# Add SCAPY_ROOT_DIR environment variable, used for tests
os.environ['SCAPY_ROOT_DIR'] = os.environ.get("PWD", os.getcwd())
except getopt.GetoptError as msg:
print("ERROR:", msg, file=sys.stderr)
raise SystemExit
autorun_func = {
Format.TEXT: scapy.autorun_get_text_interactive_session,
Format.ANSI: scapy.autorun_get_ansi_interactive_session,
Format.HTML: scapy.autorun_get_html_interactive_session,
Format.LATEX: scapy.autorun_get_latex_interactive_session,
Format.XUNIT: scapy.autorun_get_text_interactive_session,
}
if VERB > 2:
print("### Starting tests...", file=sys.stderr)
glob_output = ""
glob_result = 0
glob_title = None
UNIQUE = len(TESTFILES) == 1
# Resolve tags and asterix
for prex in six.iterkeys(copy.copy(PREEXEC_DICT)):
if "*" in prex:
pycode = PREEXEC_DICT[prex]
del PREEXEC_DICT[prex]
for gl in glob.iglob(prex):
_pycode = pycode.replace("%name%", os.path.splitext(os.path.split(gl)[1])[0]) # noqa: E501
PREEXEC_DICT[gl] = _pycode
pos_begin = 0
runned_campaigns = []
# Execute all files
for TESTFILE in TESTFILES:
if VERB > 2:
print("### Loading:", TESTFILE, file=sys.stderr)
PREEXEC = PREEXEC_DICT[TESTFILE] if TESTFILE in PREEXEC_DICT else GLOB_PREEXEC
with open(TESTFILE) as testfile:
output, result, campaign = execute_campaign(testfile, OUTPUTFILE,
PREEXEC, NUM, KW_OK, KW_KO,
DUMP, FORMAT, VERB, ONLYFAILED,
CRC, autorun_func, pos_begin,
ignore_globals)
runned_campaigns.append(campaign)
pos_begin = campaign.end_pos
if UNIQUE:
glob_title = campaign.title
glob_output += output
if not result:
glob_result = 1
if BREAKFAILED:
break
if VERB > 2:
print("### Writing output...", file=sys.stderr)
if ANNOTATIONS_MODE:
collect_types.stop()
collect_types.dump_stats("pyannotate_results")
# Concenate outputs
if FORMAT == Format.HTML:
glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL, glob_title)
# Write the final output
# Note: on Python 2, we force-encode to ignore ascii errors
# on Python 3, we need to detect the type of stream
if OUTPUTFILE == sys.stdout:
OUTPUTFILE.write(glob_output.encode("utf8", "ignore")
if 'b' in OUTPUTFILE.mode or six.PY2 else glob_output)
else:
with open(OUTPUTFILE, "wb") as f:
f.write(glob_output.encode("utf8", "ignore")
if 'b' in f.mode or six.PY2 else glob_output)
# Delete scapy's test environment vars
del os.environ['SCAPY_ROOT_DIR']
# Return state
return glob_result
def pytest_sessionfinish(session: Any, exitstatus: Any) -> None:
os.makedirs("build/", exist_ok=True)
collect_types.dump_stats("build/type_info.json")
