def target(ostream, params, parallel_tests): global PROCESSED, REPORT_LOCK, TOTAL for result in process(params, pool_size=parallel_tests): with REPORT_LOCK: PROCESSED += 1 print_progress_info(PROCESSED, TOTAL) save_result(ostream, strip_ephemeral(result))
def target(ostream, params, parallel_tests, sorted_mode): global PROCESSED, REPORT_LOCK, TOTAL for result in process(params, pool_size=parallel_tests, sorted_mode=sorted_mode): with REPORT_LOCK: PROCESSED += 1 print_progress_info(PROCESSED, TOTAL) save_result(ostream, strip_ephemeral(result))
def main(conf, istream, ostream, test_whitelist, test_blacklist, stage_whitelist, stage_blacklist, tags_whitelist, tags_blacklist, no_action, parallel_instances, parallel_tests, sorted_mode): ''' main parallel-data worker function''' global TOTAL params = dict(test_whitelist=test_whitelist, test_blacklist=test_blacklist, stage_whitelist=stage_whitelist, stage_blacklist=stage_blacklist, tags_whitelist=tags_whitelist, tags_blacklist=tags_blacklist, enabled=not no_action) params = load(istream, config_file=conf, augment=params) TOTAL = required_actions_count(params) print_progress_info(PROCESSED, TOTAL) pool = Pool(size=parallel_instances) pool.map(lambda item: target(ostream, item, parallel_tests, sorted_mode), params)
def main(conf, istream, ostream, test_whitelist, test_blacklist, stage_whitelist, stage_blacklist, tags_whitelist, tags_blacklist, no_action, parallel_instances, parallel_tests): ''' main parallel-data worker function''' global TOTAL params = dict(test_whitelist=test_whitelist, test_blacklist=test_blacklist, stage_whitelist=stage_whitelist, stage_blacklist=stage_blacklist, tags_whitelist=tags_whitelist, tags_blacklist=tags_blacklist, enabled=not no_action) params = load(istream, config_file=conf, augment=params) TOTAL = required_actions_count(params) print_progress_info(PROCESSED, TOTAL) pool = Pool(size=parallel_instances) pool.map(lambda item: target(ostream, item, parallel_tests), params)
def main(conf, istream, ostream, test_whitelist, test_blacklist, stage_whitelist, stage_blacklist, tags_whitelist, tags_blacklist, no_action): ''' main worker function performs particular stages handling generates particular stage/test result list ''' params = dict(test_whitelist=test_whitelist, test_blacklist=test_blacklist, stage_whitelist=stage_whitelist, stage_blacklist=stage_blacklist, tags_whitelist=tags_whitelist, tags_blacklist=tags_blacklist, enabled=not no_action) params = load(istream, config_file=conf, augment=params) total = required_actions_count(params) processed = 0 print_progress_info(processed, total) for item in params: for result in process(item): processed += 1 save_result(ostream, strip_ephemeral(result)) print_progress_info(processed, total)
def main(conf, istream, ostream, test_whitelist, test_blacklist, stage_whitelist, stage_blacklist, tags_whitelist, tags_blacklist, no_action): ''' main worker function performs particular stages handling generates particular stage/test result list ''' params = dict(test_whitelist=test_whitelist, test_blacklist=test_blacklist, stage_whitelist=stage_whitelist, stage_blacklist=stage_blacklist, tags_whitelist=tags_whitelist, tags_blacklist=tags_blacklist, enabled=not no_action) params = load(istream, config_file=conf, augment=params) total = required_actions_count(params) processed = 0 print_progress_info(processed, total) for item in params: for result in process(item): processed += 1 save_result(ostream, strip_ephemeral(result)) print_progress_info(processed, total)