def main(debug, verbose, config_path, parser_dir, parser_config, parser_source):
# Setup configuration
mwcp.config.load(config_path)
if parser_dir:
mwcp.config["PARSER_DIR"] = parser_dir
parser_dir = mwcp.config.get("PARSER_DIR")
if parser_config:
mwcp.config["PARSER_CONFIG_PATH"] = parser_config
parser_config = mwcp.config.get("PARSER_CONFIG_PATH")
if parser_source:
mwcp.config["PARSER_SOURCE"] = parser_source
parser_source = mwcp.config.get("PARSER_SOURCE")
# Setup logging
mwcp.setup_logging()
if debug:
logging.root.setLevel(logging.DEBUG)
elif verbose:
logging.root.setLevel(logging.INFO)
# else let log_config.yaml set log level.
# Register parsers
mwcp.register_entry_points()
if parser_dir:
mwcp.register_parser_directory(parser_dir, config_file_path=parser_config)
if parser_source:
mwcp.set_default_source(parser_source)
def main(debug, verbose, parser_dir, parser_config, parser_source):
# Setup logging
mwcp.setup_logging()
if debug:
logging.root.setLevel(logging.DEBUG)
elif verbose:
logging.root.setLevel(logging.INFO)
# else let log_config.yaml set log level.
# Register parsers
mwcp.register_entry_points()
if parser_dir:
mwcp.register_parser_directory(parser_dir, config_file_path=parser_config)
if parser_source:
mwcp.set_default_source(parser_source)
def main(debug, verbose, parser_dir, parser_config, parser_source):
# Setup logging
mwcp.setup_logging()
if debug:
logging.root.setLevel(logging.DEBUG)
elif verbose:
logging.root.setLevel(logging.INFO)
# else let log_config.yaml set log level.
# Register parsers
mwcp.register_entry_points()
if parser_dir:
mwcp.register_parser_directory(parser_dir,
config_file_path=parser_config)
if parser_source:
mwcp.set_default_source(parser_source)
def main():
import argparse
argparser = argparse.ArgumentParser()
argparser.add_argument('--parserdir', help='Parser directory to use.')
argparser.add_argument('--parserconfig', help='Parser configuration file to use')
argparser.add_argument(
'--parsersource',
help='Default parser source to use. Otherwise parsers from all sources are available.')
options = argparser.parse_args()
if options.parserdir:
mwcp.register_parser_directory(options.parserdir, config_file_path=options.parserconfig)
print('Set parser directory to: {}'.format(options.parserdir))
else:
mwcp.register_entry_points()
if options.parsersource:
mwcp.set_default_source(options.parsersource)
run(app, server='auto', host='localhost', port=8080)
def main():
import argparse
argparser = argparse.ArgumentParser()
argparser.add_argument('--parserdir', help='Parser directory to use.')
argparser.add_argument('--parserconfig',
help='Parser configuration file to use')
argparser.add_argument(
'--parsersource',
help=
'Default parser source to use. Otherwise parsers from all sources are available.'
)
options = argparser.parse_args()
if options.parserdir:
mwcp.register_parser_directory(options.parserdir,
config_file_path=options.parserconfig)
print('Set parser directory to: {}'.format(options.parserdir))
else:
mwcp.register_entry_points()
if options.parsersource:
mwcp.set_default_source(options.parsersource)
run(app, server='auto', host='localhost', port=8080)
def main(args=None):
warnings.warn(
"WARNING: mwcp-tool is deprecated. Please use \"mwcp parse\" instead.")
argparser = get_arg_parser()
args, input_files = argparser.parse_known_args(args)
# Setup logging
mwcp.setup_logging()
if args.hidedebug:
logging.root.setLevel(logging.WARNING)
elif args.debug:
logging.root.setLevel(logging.DEBUG)
# This is a preliminary check before creating the reporter to establish how output
# file prefixes should be set.
if args.disableoutputfileprefix:
args.outputfile_prefix = ''
elif args.filelistindirection or len(input_files) > 1 or any(
[os.path.isdir(x) for x in input_files]):
args.outputfile_prefix = 'md5'
if args.fields:
_print_fields(json_output=args.jsonoutput)
sys.exit(0)
# Register parsers
mwcp.register_entry_points()
if args.parserdir:
mwcp.register_parser_directory(args.parserdir,
config_file_path=args.parserconfig)
if args.parsersource:
mwcp.set_default_source(args.parsersource)
if args.list:
_print_parsers(json_output=args.jsonoutput, config_only=args.list < 2)
sys.exit(0)
if not input_files or not args.parser:
argparser.print_help()
sys.exit(0)
file_paths = _get_file_paths(input_files,
is_filelist=args.filelistindirection)
# Run MWCP
try:
reporter = mwcp.Reporter(outputdir=args.outputdir,
outputfile_prefix=args.outputfile_prefix,
tempdir=args.tempdir,
disable_output_files=args.disableoutputfiles,
disable_temp_cleanup=args.disabletempcleanup,
base64_output_files=args.base64outputfiles)
results = []
for file_path in file_paths:
result = _parse_file(reporter,
file_path,
args.parser,
include_filename=args.includefilename)
results.append(result)
if not args.jsonoutput:
reporter.print_report()
if args.csvwrite:
csv_path = args.csvwrite
_write_csv(file_paths, results, csv_path, args.base64outputfiles)
if not args.jsonoutput:
print('Wrote csv file: {}'.format(csv_path))
if args.jsonoutput:
print(json.dumps(results, indent=4))
except Exception as e:
error_message = "Error running DC3-MWCP: {}".format(e)
traceback.print_exc()
if args.jsonoutput:
print(json.dumps({'errors': [error_message]}))
else:
print(error_message)
sys.exit(1)
def main(args=None):
warnings.warn("WARNING: mwcp-tool is deprecated. Please use \"mwcp parse\" instead.")
argparser = get_arg_parser()
args, input_files = argparser.parse_known_args(args)
# Setup logging
mwcp.setup_logging()
if args.hidedebug:
logging.root.setLevel(logging.WARNING)
elif args.debug:
logging.root.setLevel(logging.DEBUG)
# This is a preliminary check before creating the reporter to establish how output
# file prefixes should be set.
if args.disableoutputfileprefix:
args.outputfile_prefix = ''
elif args.filelistindirection or len(input_files) > 1 or any([os.path.isdir(x) for x in input_files]):
args.outputfile_prefix = 'md5'
if args.fields:
_print_fields(json_output=args.jsonoutput)
sys.exit(0)
# Register parsers
mwcp.register_entry_points()
if args.parserdir:
mwcp.register_parser_directory(args.parserdir, config_file_path=args.parserconfig)
if args.parsersource:
mwcp.set_default_source(args.parsersource)
if args.list:
_print_parsers(json_output=args.jsonoutput, config_only=args.list < 2)
sys.exit(0)
if not input_files or not args.parser:
argparser.print_help()
sys.exit(0)
file_paths = _get_file_paths(input_files, is_filelist=args.filelistindirection)
# Run MWCP
try:
reporter = mwcp.Reporter(
outputdir=args.outputdir,
outputfile_prefix=args.outputfile_prefix,
tempdir=args.tempdir,
disable_output_files=args.disableoutputfiles,
disable_temp_cleanup=args.disabletempcleanup,
base64_output_files=args.base64outputfiles)
results = []
for file_path in file_paths:
result = _parse_file(
reporter, file_path, args.parser, include_filename=args.includefilename)
results.append(result)
if not args.jsonoutput:
reporter.print_report()
if args.csvwrite:
csv_path = args.csvwrite
_write_csv(file_paths, results, csv_path, args.base64outputfiles)
if not args.jsonoutput:
print('Wrote csv file: {}'.format(csv_path))
if args.jsonoutput:
print(json.dumps(results, indent=4))
except Exception as e:
error_message = "Error running DC3-MWCP: {}".format(e)
traceback.print_exc()
if args.jsonoutput:
print(json.dumps({'errors': [error_message]}))
else:
print(error_message)
sys.exit(1)
def main():
"""Run tool."""
warnings.warn("WARNING: mwcp-test is deprecated. Please use \"mwcp test\" instead.")
print('')
# Get command line arguments
argparser = get_arg_parser()
args, input_files = argparser.parse_known_args()
# Setup logging
mwcp.setup_logging()
logging.root.setLevel(logging.ERROR - (args.verbose * 10))
if args.all_tests or not args.parser_name:
parsers = [None]
else:
parsers = [args.parser_name]
if args.parserdir and args.parserconfig:
mwcp.register_parser_directory(args.parserdir, args.parserconfig)
elif args.parserdir or args.parserconfig:
raise ValueError('Both --parserdir and --parserconfig must be specified.')
else:
mwcp.register_entry_points()
if args.parsersource:
mwcp.set_default_source(args.parsersource)
# Configure reporter based on args
reporter = mwcp.Reporter(disable_output_files=True)
# Configure test object
tester = Tester(
reporter=reporter, results_dir=args.test_case_dir, parser_names=parsers, nprocs=args.nprocs,
field_names=filter(None, args.field_names.split(",")),
ignore_field_names=filter(None, args.exclude_field_names.split(","))
)
# Gather all our input files
if args.input_file:
input_files = read_input_list(input_files[0])
# Delete files from test cases
if args.delete:
removed_files = tester.remove_test_results(
args.parser_name, input_files)
for filename in removed_files:
print(u"Removing results for {} in {}".format(
filename, tester.get_results_filepath(args.parser_name)))
# Update previously existing test cases
elif args.update and args.parser_name:
print("Updating test cases. May take a while...")
results_file_path = tester.get_results_filepath(args.parser_name)
if os.path.isfile(results_file_path):
input_files = tester.list_test_files(args.parser_name)
else:
sys.exit(u"No test case file found for parser '{}'. "
u"No update could be made.".format(args.parser_name))
update_tests(tester, input_files, args.parser_name)
# Add/update test cases for specified input files and specified parser
elif args.parser_name and not args.delete and input_files:
update_tests(tester, input_files, args.parser_name)
# Run test cases
else:
print("Running test cases. May take a while...")
start_time = timeit.default_timer()
test_results = []
all_passed = True
total = tester.total
failed = []
# Generate format string.
digits = len(str(total))
if not tester.test_cases:
parser_len = 10
filename_len = 10
else:
parser_len = max(len(test_case.parser_name) for test_case in tester.test_cases)
filename_len = max(len(test_case.filename) for test_case in tester.test_cases)
msg_format = "{{parser:{0}}} {{filename:{1}}} {{run_time:.4f}}s".format(parser_len, filename_len)
format_str = "{{count:> {0}d}}/{{total:0{0}d}} - ".format(digits) + msg_format
# Run tests and output progress results.
for count, test_result in enumerate(tester, start=1):
all_passed &= test_result.passed
if not test_result.passed:
failed.append((count, test_result.parser_name, test_result.filename))
if test_result.run_time: # Ignore missing tests from stat summary.
test_results.append(test_result)
if not args.silent:
message = format_str.format(
count=count,
total=total,
parser=test_result.parser_name,
filename=test_result.filename,
run_time=test_result.run_time
)
# Skip print() to immediately flush stdout buffer (issue in Docker containers)
sys.stdout.write(message + '\n')
sys.stdout.flush()
test_result.print(
failed_tests=True, passed_tests=not args.only_failed_tests, json_format=args.json
)
end_time = timeit.default_timer()
# Present test statistics
if not args.silent and test_results:
print('\nTest stats:')
print('\nTop 10 Slowest Test Cases:')
format_str = "{index:2}. " + msg_format
# Cases sorted slowest first
sorted_cases = sorted(test_results, key=lambda x: x.run_time, reverse=True)
for i, test_result in enumerate(sorted_cases[:10], start=1):
print(format_str.format(
index=i,
parser=test_result.parser_name,
filename=test_result.filename,
run_time=test_result.run_time
))
print('\nTop 10 Fastest Test Cases:')
for i, test_result in enumerate(list(reversed(sorted_cases))[:10], start=1):
print(format_str.format(
index=i,
parser=test_result.parser_name,
filename=test_result.filename,
run_time=test_result.run_time
))
run_times = [test_result.run_time for test_result in test_results]
print('\nMean Running Time: {:.4f}s'.format(
sum(run_times) / len(test_results)
))
print('Median Running Time: {:.4f}s'.format(
_median(run_times)
))
print('Cumulative Running Time: {}'.format(datetime.timedelta(seconds=sum(run_times))))
print()
print("Total Running Time: {}".format(datetime.timedelta(seconds=end_time - start_time)))
if failed:
print()
print("Failed tests:")
for test_info in failed:
print("#{} - {}\t{}".format(*test_info))
print()
print("All Passed = {0}\n".format(all_passed))
exit(0 if all_passed else 1)
def register():
global report
mwcp.register_entry_points()
mwcp.register_parser_directory(MWCP_PARSERS_DIR_PATH)
report = mwcp.Report(output_directory=os.getcwd())
return report
return output
def main():
import argparse
argparser = argparse.ArgumentParser()
argparser.add_argument('--parserdir', help='Parser directory to use.')
argparser.add_argument('--parserconfig', help='Parser configuration file to use')
argparser.add_argument(
'--parsersource',
help='Default parser source to use. Otherwise parsers from all sources are available.')
options = argparser.parse_args()
if options.parserdir:
mwcp.register_parser_directory(options.parserdir, config_file_path=options.parserconfig)
print('Set parser directory to: {}'.format(options.parserdir))
else:
mwcp.register_entry_points()
if options.parsersource:
mwcp.set_default_source(options.parsersource)
run(app, server='auto', host='localhost', port=8080)
if __name__ == '__main__':
main()
else:
mwcp.register_entry_points()
application = app
import argparse
argparser = argparse.ArgumentParser()
argparser.add_argument('--parserdir', help='Parser directory to use.')
argparser.add_argument('--parserconfig',
help='Parser configuration file to use')
argparser.add_argument(
'--parsersource',
help=
'Default parser source to use. Otherwise parsers from all sources are available.'
)
options = argparser.parse_args()
if options.parserdir:
mwcp.register_parser_directory(options.parserdir,
config_file_path=options.parserconfig)
print('Set parser directory to: {}'.format(options.parserdir))
else:
mwcp.register_entry_points()
if options.parsersource:
mwcp.set_default_source(options.parsersource)
run(app, server='auto', host='localhost', port=8080)
if __name__ == '__main__':
main()
else:
mwcp.register_entry_points()
application = app
def init_app(app):
"""Initialize the Flask application and MWCP"""
# Initialize MWCP
mwcp.config.load(os.getenv("MWCP_CONFIG", None))
mwcp.register_entry_points()
