def main(args=dict()):
log.config_logging(getattr(logging, 'INFO', None))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('Classifier'))
repo_list, pipeline = _validate_input(args)
filter_output_dir = os.path.join(os.path.dirname(__file__),
'../pair-filter/output-json/')
if pipeline and not os.path.exists(filter_output_dir):
log.error(
'pipeline == true, but output_file_path ({}) does not exist. '
'Exiting PairClassifier.'.format(filter_output_dir))
return
for repo in repo_list:
if pipeline:
task_name = repo.replace('/', '-')
json_path = os.path.join(filter_output_dir, task_name + '.json')
if not os.path.exists(json_path):
log.error(json_path, 'does not exist. Repo', repo,
'will be skipped.')
continue
# Get the input json from the file generated by pair-filter.
dir_of_jsons = generate_build_pair_json(repo, json_path)
else:
# Get the input json from the DB.
dir_of_jsons = generate_build_pair_json(repo)
PairClassifier.run(repo, dir_of_jsons, args)
def main(argv=None):
argv = argv or sys.argv
# Configure logging.
log.config_logging(getattr(logging, 'INFO', None))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('ReproducedResultsAnalyzer'))
input_file, runs, task_name = _validate_input(argv)
ReproducedResultsAnalyzer(input_file, runs, task_name).run()
def main(args=dict()):
log.config_logging(getattr(logging, 'INFO', None))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('Classifier'))
repo_list = _validate_input(args)
for repo in repo_list:
# get the input json from DB
dir_of_jsons = generate_build_pair_json(repo)
PairClassifier.run(repo, dir_of_jsons, args)
def main(argv=None):
argv = argv or sys.argv
# Configure logging.
log.config_logging(getattr(logging, 'INFO', None))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('MetadataPackager'))
# Decrease logging severity from the requests library.
logging.getLogger('requests').setLevel(logging.WARNING)
input_file, csv_mode = _validate_input(argv)
Packager(input_file, csv_mode).run()
def main(argv=None):
argv = argv or sys.argv
# Configure logging.
log.config_logging(getattr(logging, 'INFO', None))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('PairFilter'))
if not path.exists(DOCKERHUB_IMAGES_JSON):
log.info(
'File dockerhub_image.json not found. Please run gen_image_list.py'
)
repo, dir_of_jsons = _validate_input(argv)
PairFilter.run(repo, dir_of_jsons)
def main(argv=None):
argv = argv or sys.argv
# Configure logging.
log.config_logging(getattr(logging, 'INFO', None))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('Analyzer'))
mode, reproduced, orig, log_filename, print_result, job_id, build_system, trigger_sha, repo = _validate_input(argv)
analyzer = Analyzer()
if mode == 0:
analyzer.compare_single_log(reproduced, orig, job_id, build_system, trigger_sha, repo, print_result)
elif mode == 1:
analyzer.analyze_single_log(log_filename, job_id, build_system, trigger_sha, repo, print_result)
else:
raise Exception('Unsupported mode: {}.'.format(mode))
def main(argv=None):
argv = argv or sys.argv
# Configure logging.
log.config_logging(getattr(logging, 'INFO', None))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('Reproducer'))
# Parse input.
shortopts = 'i:t:o:kpds'
longopts = 'input-file= threads= task-name= keep package skip-check-disk'.split(
)
input_file = None
threads = 1
task_name = None
keep = False
package_mode = False
dependency_solver = False
skip_check_disk = False
try:
optlist, args = getopt.getopt(argv[1:], shortopts, longopts)
except getopt.GetoptError:
log.error('Error parsing arguments. Exiting.')
print_usage()
sys.exit(2)
for opt, arg in optlist:
if opt in ['-i', '--input-file']:
input_file = arg
if opt in ['-t', '--threads']:
threads = int(arg)
if opt in ['-o', '--task-name']:
task_name = arg
if opt in ['-k', '--keep']:
keep = True
if opt in ['-p', '--package']:
package_mode = True
if opt in ['-d', '--dependency-solver']:
dependency_solver = True
if opt in ['-s', '--skip-check-disk']:
skip_check_disk = True
if not input_file:
print_usage()
sys.exit(2)
if threads <= 0:
log.error('The threads argument must be greater than 0. Exiting.')
sys.exit(1)
if not os.path.isfile(input_file):
log.error(
'The input_file argument is not a file or does not exist. Exiting.'
)
sys.exit(1)
if not task_name:
print_usage()
sys.exit(2)
# Initialize JobDispatcher.
if package_mode:
reproducer = ImagePackager(input_file, task_name, threads, keep,
package_mode, dependency_solver,
skip_check_disk)
else:
reproducer = JobReproducer(input_file, task_name, threads, keep,
package_mode, dependency_solver,
skip_check_disk)
reproducer.run()
def _thread_main(repo, task_name, log_level, skip_if_output_exists,
keep_clone):
log.config_logging(log_level, Utils.log_file_path_from_repo(repo))
# Log the current version of this BugSwarm component.
log.info(get_current_component_version_message('PairFinder'))
log.info('Processing', repo)
output_file_path = Utils.output_file_path_from_repo(repo, task_name)
if skip_if_output_exists and os.path.exists(
output_file_path) and os.path.getsize(output_file_path) > 0:
log.info('Skipping', repo, 'because output already exists.')
return
start_time = time.time()
in_context = {
'repo': repo,
'utils': Utils(),
'keep_clone': keep_clone,
'task_name': task_name,
'mined_project_builder': MinedProjectBuilder(),
}
steps = [
Preflight(),
GetJobsFromTravisAPI(),
GroupJobsByBranch(),
ExtractAllBuildPairs(),
AlignJobPairs(),
GetPullRequestMergeStatuses(),
DownloadPullRequestCommits(),
AssignTriggerCommits(),
AssignBaseCommits(),
CleanPairs(),
GetBuildSystemInfo(),
Postflight(),
]
pipeline = Pipeline(steps)
result, out_context = pipeline.run(None, in_context)
builder = out_context['mined_project_builder']
builder.repo = repo
builder.latest_mined_version = Utils.get_latest_commit_for_repo(repo)
(mined_build_pairs, mined_job_pairs, mined_pr_build_pairs,
mined_pr_job_pairs) = Utils.count_mined_pairs_in_branches(result)
builder.mined_job_pairs = mined_job_pairs
builder.mined_pr_job_pairs = mined_pr_job_pairs
builder.mined_build_pairs = mined_build_pairs
builder.mined_pr_build_pairs = mined_pr_build_pairs
mined_project = builder.build()
OutputManager.output_to_database(mined_project)
if not result:
# A filter in the pipeline encountered a fatal error and made the pipeline exit early.
# Skip writing the output file.
return
OutputManager.output(repo, output_path=output_file_path, branches=result)
elapsed = time.time() - start_time
log.info('Processed {} in {} seconds. Done!'.format(repo, elapsed))