def pytest_sessionfinish(session): plugin_base.final_process_cleanup() if session.config.option.dump_pyannotate: from pyannotate_runtime import collect_types collect_types.dump_stats(session.config.option.dump_pyannotate)
def main(): collect_types.init_types_collection() with collect_types.collect(): os.chdir('/home/maciej/Desktop/REPO') cmd.launch(['status']) cmd.launch(['file']) cmd.launch(['format']) cmd.launch(['help']) os.system('rm -f .git/machete') cmd.launch(['add', '--yes']) cmd.launch(['anno']) cmd.launch(['discover', '--yes']) # # cmd.launch(['e']) cmd.launch(['delete-unmanaged']) cmd.launch(['g','f']) cmd.launch(['list','managed']) cmd.launch(['show','f']) # # # cmd.launch(['d']) cmd.launch(['fork-point']) # # # cmd.launch(['l']) cmd.launch(['traverse']) # # # cmd.launch(['update']) os.chdir(file_dir) collect_types.dump_stats(json_name) if DEBUG == True: if os.path.exists(json_name): print('dziala!!) # del_json() else: print(':C') else: create_types() print('Fajno')
def main(): loader = unittest.TestLoader() g = glob.glob py = g('frontend/*_test.py') + g('osh/*_test.py') + g( 'core/*_test.py') + g('') # hangs py.remove('core/process_test.py') modules = [] for p in py: mod_name = p[:-3].replace('/', '.') print(mod_name) modules.append(__import__(mod_name, fromlist=['.'])) for m in modules: print(m) suites = [loader.loadTestsFromModule(m) for m in modules] suite = unittest.TestSuite() for s in suites: suite.addTest(s) runner = unittest.TextTestRunner() collect_types.init_types_collection() with collect_types.collect(): runner.run(suite) if 0: TopLevel() Match() Arith() collect_types.dump_stats('type_info.json')
def main_pyannotate(): try: collect_types.init_types_collection() collect_types.resume() return main() finally: log.info("dumping type_info") collect_types.pause() collect_types.dump_stats('type_info.json')
def main(): import sys from pyannotate_runtime import collect_types collect_types.init_types_collection() with collect_types.collect(): dog = Dog('sharik') dog.name dog.bark('ru') collect_types.dump_stats('type_info.json')
def pytest_sessionfinish(session): from sqlalchemy.testing import asyncio asyncio._maybe_async_provisioning(plugin_base.final_process_cleanup) if session.config.option.dump_pyannotate: from pyannotate_runtime import collect_types collect_types.dump_stats(session.config.option.dump_pyannotate)
def main_annotate(): from pyannotate_runtime import collect_types collect_types.init_types_collection(filter_filename=filter_filename) collect_types.resume() try: manager.run() except BaseException: pass collect_types.dump_stats(b"type_info.json")
def annotations_collect(): """Fixture to collect 'annotations' for pyannonate Yields: [None]: [Generates file annotations.txt] """ # pipx install pyannotate==1.2.0 from pyannotate_runtime import collect_types collect_types.init_types_collection() collect_types.start() yield None collect_types.stop() collect_types.dump_stats("annotations.txt")
def test_input(self, capsys): collect_types.init_types_collection() with collect_types.collect(): assert len(sqalp.known_formats) > 1 assert sqalp.get_session(logging.ERROR) session = sqalp.get_session(logging.DEBUG) parser = sqalp.get_parser(sqalp.known_formats['common']) sqalp.file_import(session, open(f'{temp}/sample.log', 'r'), parser, False) results = sqalp.get_by_date(session) assert results != TestClass.count_json results = sqalp.get_by_date_by_ua(session) assert results != TestClass.ua_json results = sqalp.get_by_date_verb_ratio(session) assert results != TestClass.ration_json output_data = OrderedDict([('2011-12-01', [2822]), ('2011-12-02', [2572]), ('2011-12-03', [604])]) output_print_data_default = \ "+--------------+--------------+--------------+\n" \ "| 2011-12-01 | 2011-12-02 | 2011-12-03 |\n" \ "+==============+==============+==============+\n" \ "| 2822 | 2572 | 604 |\n" \ "+--------------+--------------+--------------+\n" output_print_data_json = \ '{"2011-12-01": [2822], ' \ '"2011-12-02": [2572], ' \ '"2011-12-03": [604]}' sqalp.output(output_data, 'grid') captured = capsys.readouterr() assert output_print_data_default in captured.out sqalp.output(output_data, 'json') captured = capsys.readouterr() assert output_print_data_json in captured.out sqalp.setup_logging(logging.ERROR) # Write collectd type data so it can be imported into source # Not in PWD ot tox/setup get miffed collect_types.dump_stats('/tmp/types.py')
def main(): argv = sys.argv[1:] logger = logging.getLogger("scapy") logger.addHandler(logging.StreamHandler()) import scapy print(dash + " UTScapy - Scapy %s - %s" % (scapy.__version__, sys.version.split(" ")[0])) # Parse arguments FORMAT = Format.ANSI OUTPUTFILE = sys.stdout LOCAL = 0 NUM = None NON_ROOT = False KW_OK = [] KW_KO = [] DUMP = 0 DOCS = 0 CRC = True BREAKFAILED = True ONLYFAILED = False VERB = 3 GLOB_PREEXEC = "" PREEXEC_DICT = {} MODULES = [] TESTFILES = [] ANNOTATIONS_MODE = False INTERPRETER = False try: opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DRdCiFqNP:s:x") for opt, optarg in opts[0]: if opt == "-h": usage() elif opt == "-b": BREAKFAILED = False elif opt == "-F": ONLYFAILED = True elif opt == "-q": VERB -= 1 elif opt == "-D": DUMP = 2 elif opt == "-R": DOCS = 1 elif opt == "-d": DUMP = 1 elif opt == "-C": CRC = False elif opt == "-i": INTERPRETER = True elif opt == "-x": ANNOTATIONS_MODE = True elif opt == "-P": GLOB_PREEXEC += "\n" + optarg elif opt == "-f": try: FORMAT = Format.from_string(optarg) except KeyError as msg: raise getopt.GetoptError("Unknown output format %s" % msg) elif opt == "-t": TESTFILES.append(optarg) TESTFILES = resolve_testfiles(TESTFILES) elif opt == "-T": TESTFILES.remove(optarg) elif opt == "-c": data = parse_config_file(optarg, VERB) BREAKFAILED = data.breakfailed ONLYFAILED = data.onlyfailed VERB = data.verb DUMP = data.dump CRC = data.crc PREEXEC_DICT = data.preexec GLOB_PREEXEC = data.global_preexec OUTPUTFILE = data.outfile TESTFILES = data.testfiles LOCAL = 1 if data.local else 0 NUM = data.num MODULES = data.modules KW_OK.extend(data.kw_ok) KW_KO.extend(data.kw_ko) try: FORMAT = Format.from_string(data.format) except KeyError as msg: raise getopt.GetoptError("Unknown output format %s" % msg) TESTFILES = resolve_testfiles(TESTFILES) for testfile in resolve_testfiles(data.remove_testfiles): try: TESTFILES.remove(testfile) except ValueError: error_m = "Cannot remove %s from test files" % testfile raise getopt.GetoptError(error_m) elif opt == "-o": OUTPUTFILE = optarg if not os.access(os.path.dirname(os.path.abspath(OUTPUTFILE)), os.W_OK): raise getopt.GetoptError("Cannot write to file %s" % OUTPUTFILE) elif opt == "-l": LOCAL = 1 elif opt == "-n": NUM = [] for v in (x.strip() for x in optarg.split(",")): try: NUM.append(int(v)) except ValueError: v1, v2 = [int(e) for e in v.split('-', 1)] NUM.extend(range(v1, v2 + 1)) elif opt == "-N": NON_ROOT = True elif opt == "-m": MODULES.append(optarg) elif opt == "-k": KW_OK.extend(optarg.split(",")) elif opt == "-K": KW_KO.extend(optarg.split(",")) except getopt.GetoptError as msg: print("ERROR:", msg) raise SystemExit if FORMAT in [Format.LIVE, Format.ANSI]: theme = DefaultTheme() else: theme = BlackAndWhite() # Disable tests if needed # Discard Python3 tests when using Python2 if six.PY2: KW_KO.append("python3_only") if VERB > 2: print(" " + arrow + " Python 2 mode") try: if NON_ROOT or os.getuid() != 0: # Non root # Discard root tests KW_KO.append("needs_root") if VERB > 2: print(" " + arrow + " Non-root mode") except AttributeError: pass if conf.use_pcap or WINDOWS: KW_KO.append("not_libpcap") if VERB > 2: print(" " + arrow + " libpcap mode") KW_KO.append("disabled") # Process extras if six.PY3: KW_KO.append("FIXME_py3") if ANNOTATIONS_MODE: try: from pyannotate_runtime import collect_types except ImportError: raise ImportError("Please install pyannotate !") collect_types.init_types_collection() collect_types.start() if VERB > 2: print(" " + arrow + " Booting scapy...") try: from scapy import all as scapy except Exception as e: print("[CRITICAL]: Cannot import Scapy: %s" % e) traceback.print_exc() sys.exit(1) # Abort the tests for m in MODULES: try: mod = import_module(m) six.moves.builtins.__dict__.update(mod.__dict__) except ImportError as e: raise getopt.GetoptError("cannot import [%s]: %s" % (m, e)) autorun_func = { Format.TEXT: scapy.autorun_get_text_interactive_session, Format.ANSI: scapy.autorun_get_ansi_interactive_session, Format.HTML: scapy.autorun_get_html_interactive_session, Format.LATEX: scapy.autorun_get_latex_interactive_session, Format.XUNIT: scapy.autorun_get_text_interactive_session, Format.LIVE: scapy.autorun_get_live_interactive_session, } if VERB > 2: print(" " + arrow + " Discovering tests files...") glob_output = "" glob_result = 0 glob_title = None UNIQUE = len(TESTFILES) == 1 # Resolve tags and asterix for prex in six.iterkeys(copy.copy(PREEXEC_DICT)): if "*" in prex: pycode = PREEXEC_DICT[prex] del PREEXEC_DICT[prex] for gl in glob.iglob(prex): _pycode = pycode.replace( "%name%", os.path.splitext(os.path.split(gl)[1])[0]) # noqa: E501 PREEXEC_DICT[gl] = _pycode pos_begin = 0 runned_campaigns = [] from scapy.main import _scapy_builtins scapy_ses = _scapy_builtins() import_UTscapy_tools(scapy_ses) # Execute all files for TESTFILE in TESTFILES: if VERB > 2: print(theme.green(dash + " Loading: %s" % TESTFILE)) PREEXEC = PREEXEC_DICT[ TESTFILE] if TESTFILE in PREEXEC_DICT else GLOB_PREEXEC with open(TESTFILE) as testfile: output, result, campaign = execute_campaign( testfile, OUTPUTFILE, PREEXEC, NUM, KW_OK, KW_KO, DUMP, DOCS, FORMAT, VERB, ONLYFAILED, CRC, INTERPRETER, autorun_func, theme, pos_begin=pos_begin, scapy_ses=copy.copy(scapy_ses)) runned_campaigns.append(campaign) pos_begin = campaign.end_pos if UNIQUE: glob_title = campaign.title glob_output += output if not result: glob_result = 1 if BREAKFAILED: break if VERB > 2: print(checkmark + " All campaigns executed. Writing output...") if ANNOTATIONS_MODE: collect_types.stop() collect_types.dump_stats("pyannotate_results") # Concenate outputs if FORMAT == Format.HTML: glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL, glob_title) if FORMAT == Format.LATEX: glob_output = pack_latex_campaigns(runned_campaigns, glob_output, LOCAL, glob_title) # Write the final output # Note: on Python 2, we force-encode to ignore ascii errors # on Python 3, we need to detect the type of stream if OUTPUTFILE == sys.stdout: print(glob_output, file=OUTPUTFILE) else: with open(OUTPUTFILE, "wb") as f: f.write( glob_output.encode("utf8", "ignore") if 'b' in f.mode or six.PY2 else glob_output) # Print end message if VERB > 2: if glob_result == 0: print(theme.green("UTscapy ended successfully")) else: print(theme.red("UTscapy ended with error code %s" % glob_result)) # Check active threads if VERB > 2: if threading.active_count() > 1: print("\nWARNING: UNFINISHED THREADS") print(threading.enumerate()) import multiprocessing processes = multiprocessing.active_children() if processes: print("\nWARNING: UNFINISHED PROCESSES") print(processes) sys.stdout.flush() # Return state return glob_result
def pytest_sessionfinish(session, exitstatus): os.makedirs('build/', exist_ok=True) collect_types.dump_stats("build/type_info.json")
def report(self, stream): collect_types.dump_stats('type_info.json')
import pytest from pyannotate_runtime import collect_types if __name__ == '__main__': collect_types.init_types_collection() with collect_types.collect(): pytest.main() collect_types.dump_stats('type_info.json')
def pytest_sessionfinish(session, exitstatus): collect_types.dump_stats("annotations.json")
# Called from Python-2.7.13/Modules/main.c. def _cpython_main_hook(): sys.exit(main(sys.argv)) if __name__ == '__main__': pyann_out = posix.environ.get('PYANN_OUT') if pyann_out: from pyannotate_runtime import collect_types collect_types.init_types_collection() with collect_types.collect(): status = main(sys.argv) collect_types.dump_stats(pyann_out) sys.exit(status) elif posix.environ.get('RESOLVE') == '1': from opy import resolve resolve.Walk(dict(sys.modules)) elif posix.environ.get('CALLGRAPH') == '1': # NOTE: This could end up as opy.InferTypes(), opy.GenerateCode(), etc. from opy import callgraph callgraph.Walk(main, sys.modules) else: sys.exit(main(sys.argv))
def pytest_sessionfinish(session, exitstatus): from pyannotate_runtime import collect_types collect_types.dump_stats("type_info.json")
def pytest_sessionfinish(session, exitstatus): collect_types.dump_stats("type_info.json")
def pytest_unconfigure(self, config): """Unconfigure the pytest plugin. Happens when pytest is about to exit.""" collect_types.dump_stats(self.output_file)
from .patterns.constants import DataSetType from .reader import download, process_data, save_data num_cpus = os.cpu_count() or 4 # Make sure workers don't start a copy ray.shutdown() ray.init(num_cpus=num_cpus, include_dashboard=False) def main(): # download() for v in DataSetType: data_set_type = v.value print(f"[INFO] Processing {data_set_type} set") processed = process_data(data_set_type, parallel=False) save_data(processed, data_set_type) if __name__ == "__main__": # Collect runtime type data for pyannotate collect_types.init_types_collection() with collect_types.collect(): main() collect_types.dump_stats("./pyannotate_runtime.stats")
def main(): argv = sys.argv[1:] ignore_globals = list(six.moves.builtins.__dict__) # Parse arguments FORMAT = Format.ANSI OUTPUTFILE = sys.stdout LOCAL = 0 NUM = None KW_OK = [] KW_KO = [] DUMP = 0 CRC = True BREAKFAILED = True ONLYFAILED = False VERB = 3 GLOB_PREEXEC = "" PREEXEC_DICT = {} SCAPY = "scapy" MODULES = [] TESTFILES = [] ANNOTATIONS_MODE = False try: opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DdCFqP:s:x") for opt, optarg in opts[0]: if opt == "-h": usage() elif opt == "-b": BREAKFAILED = False elif opt == "-F": ONLYFAILED = True elif opt == "-q": VERB -= 1 elif opt == "-D": DUMP = 2 elif opt == "-d": DUMP = 1 elif opt == "-C": CRC = False elif opt == "-x": ANNOTATIONS_MODE = True elif opt == "-s": SCAPY = optarg elif opt == "-P": GLOB_PREEXEC += "\n" + optarg elif opt == "-f": try: FORMAT = Format.from_string(optarg) except KeyError as msg: raise getopt.GetoptError("Unknown output format %s" % msg) elif opt == "-t": TESTFILES.append(optarg) TESTFILES = resolve_testfiles(TESTFILES) elif opt == "-T": TESTFILES.remove(optarg) elif opt == "-c": data = parse_config_file(optarg, VERB) BREAKFAILED = data.breakfailed ONLYFAILED = data.onlyfailed VERB = data.verb DUMP = data.dump CRC = data.crc SCAPY = data.scapy PREEXEC_DICT = data.preexec GLOB_PREEXEC = data.global_preexec OUTPUTFILE = data.outfile TESTFILES = data.testfiles LOCAL = 1 if data.local else 0 NUM = data.num MODULES = data.modules KW_OK.extend(data.kw_ok) KW_KO.extend(data.kw_ko) try: FORMAT = Format.from_string(data.format) except KeyError as msg: raise getopt.GetoptError("Unknown output format %s" % msg) TESTFILES = resolve_testfiles(TESTFILES) for testfile in resolve_testfiles(data.remove_testfiles): try: TESTFILES.remove(testfile) except ValueError: error_m = "Cannot remove %s from test files" % testfile raise getopt.GetoptError(error_m) elif opt == "-o": OUTPUTFILE = optarg if not os.access(os.path.dirname(os.path.abspath(OUTPUTFILE)), os.W_OK): raise getopt.GetoptError("Cannot write to file %s" % OUTPUTFILE) elif opt == "-l": LOCAL = 1 elif opt == "-n": NUM = [] for v in (x.strip() for x in optarg.split(",")): try: NUM.append(int(v)) except ValueError: v1, v2 = [int(e) for e in v.split('-', 1)] NUM.extend(range(v1, v2 + 1)) elif opt == "-m": MODULES.append(optarg) elif opt == "-k": KW_OK.extend(optarg.split(",")) elif opt == "-K": KW_KO.extend(optarg.split(",")) # Discard Python3 tests when using Python2 if six.PY2: KW_KO.append("python3_only") if ANNOTATIONS_MODE: try: from pyannotate_runtime import collect_types except ImportError: raise ImportError("Please install pyannotate !") collect_types.init_types_collection() collect_types.start() if VERB > 2: print("### Booting scapy...", file=sys.stderr) try: from scapy import all as scapy except Exception as e: print("[CRITICAL]: Cannot import Scapy: %s" % e, file=sys.stderr) traceback.print_exc() sys.exit(1) # Abort the tests for m in MODULES: try: mod = import_module(m) six.moves.builtins.__dict__.update(mod.__dict__) except ImportError as e: raise getopt.GetoptError("cannot import [%s]: %s" % (m, e)) if WINDOWS: from scapy.arch.windows import route_add_loopback route_add_loopback() # Add SCAPY_ROOT_DIR environment variable, used for tests os.environ['SCAPY_ROOT_DIR'] = os.environ.get("PWD", os.getcwd()) except getopt.GetoptError as msg: print("ERROR:", msg, file=sys.stderr) raise SystemExit autorun_func = { Format.TEXT: scapy.autorun_get_text_interactive_session, Format.ANSI: scapy.autorun_get_ansi_interactive_session, Format.HTML: scapy.autorun_get_html_interactive_session, Format.LATEX: scapy.autorun_get_latex_interactive_session, Format.XUNIT: scapy.autorun_get_text_interactive_session, } if VERB > 2: print("### Starting tests...", file=sys.stderr) glob_output = "" glob_result = 0 glob_title = None UNIQUE = len(TESTFILES) == 1 # Resolve tags and asterix for prex in six.iterkeys(copy.copy(PREEXEC_DICT)): if "*" in prex: pycode = PREEXEC_DICT[prex] del PREEXEC_DICT[prex] for gl in glob.iglob(prex): _pycode = pycode.replace("%name%", os.path.splitext(os.path.split(gl)[1])[0]) # noqa: E501 PREEXEC_DICT[gl] = _pycode pos_begin = 0 runned_campaigns = [] # Execute all files for TESTFILE in TESTFILES: if VERB > 2: print("### Loading:", TESTFILE, file=sys.stderr) PREEXEC = PREEXEC_DICT[TESTFILE] if TESTFILE in PREEXEC_DICT else GLOB_PREEXEC with open(TESTFILE) as testfile: output, result, campaign = execute_campaign(testfile, OUTPUTFILE, PREEXEC, NUM, KW_OK, KW_KO, DUMP, FORMAT, VERB, ONLYFAILED, CRC, autorun_func, pos_begin, ignore_globals) runned_campaigns.append(campaign) pos_begin = campaign.end_pos if UNIQUE: glob_title = campaign.title glob_output += output if not result: glob_result = 1 if BREAKFAILED: break if VERB > 2: print("### Writing output...", file=sys.stderr) if ANNOTATIONS_MODE: collect_types.stop() collect_types.dump_stats("pyannotate_results") # Concenate outputs if FORMAT == Format.HTML: glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL, glob_title) # Write the final output # Note: on Python 2, we force-encode to ignore ascii errors # on Python 3, we need to detect the type of stream if OUTPUTFILE == sys.stdout: OUTPUTFILE.write(glob_output.encode("utf8", "ignore") if 'b' in OUTPUTFILE.mode or six.PY2 else glob_output) else: with open(OUTPUTFILE, "wb") as f: f.write(glob_output.encode("utf8", "ignore") if 'b' in f.mode or six.PY2 else glob_output) # Delete scapy's test environment vars del os.environ['SCAPY_ROOT_DIR'] # Return state return glob_result
def pytest_sessionfinish(session: Any, exitstatus: Any) -> None: os.makedirs("build/", exist_ok=True) collect_types.dump_stats("build/type_info.json")