def get_revisions_list(bucket_path, testcase=None):
"""Returns a sorted ascending list of revisions from a bucket path, excluding
bad build revisions and testcase crash revision (if any)."""
revision_pattern = revisions.revision_pattern_from_build_bucket_path(
bucket_path)
revision_urls = get_build_urls_list(bucket_path, reverse=False)
if not revision_urls:
return None
# Parse the revisions out of the build urls.
revision_list = []
for url in revision_urls:
match = re.match(revision_pattern, url)
if match:
revision = revisions.convert_revision_to_integer(match.group(1))
revision_list.append(revision)
# Remove revisions for bad builds from the revision list.
job_type = environment.get_value('JOB_NAME')
bad_builds = ndb_utils.get_all_from_query(
data_types.BuildMetadata.query(
ndb_utils.is_true(data_types.BuildMetadata.bad_build),
data_types.BuildMetadata.job_type == job_type))
for bad_build in bad_builds:
# Don't remove testcase revision even if it is in bad build list. This
# usually happens when a bad bot sometimes marks a particular revision as
# bad due to flakiness.
if testcase and bad_build.revision == testcase.crash_revision:
continue
if bad_build.revision in revision_list:
revision_list.remove(bad_build.revision)
return revision_list
def setup_trunk_build():
"""Sets up latest trunk build."""
release_build_bucket_path = environment.get_value('RELEASE_BUILD_BUCKET_PATH')
sym_release_build_bucket_path = environment.get_value(
'SYM_RELEASE_BUILD_BUCKET_PATH')
sym_debug_build_bucket_path = environment.get_value(
'SYM_DEBUG_BUILD_BUCKET_PATH')
base_build_dir = _base_build_dir(release_build_bucket_path)
_setup_build_directories(base_build_dir)
release_build_urls = get_build_urls_list(release_build_bucket_path)
if not release_build_urls:
logs.log_error('Error getting list of release build urls from %s.' %
release_build_bucket_path)
return None
other_build_url_lists = []
if sym_release_build_bucket_path:
sym_release_build_urls = get_build_urls_list(sym_release_build_bucket_path)
if not sym_release_build_urls:
logs.log_error(
'Error getting list of symbolized release build urls from %s.' %
sym_release_build_bucket_path)
return None
other_build_url_lists.append((sym_release_build_bucket_path,
sym_release_build_urls))
if sym_debug_build_bucket_path:
sym_debug_builds_urls = get_build_urls_list(sym_debug_build_bucket_path)
if not sym_debug_builds_urls:
logs.log_error(
'Error getting list of symbolized debug build urls from %s.' %
sym_debug_build_bucket_path)
return None
other_build_url_lists.append((sym_debug_build_bucket_path,
sym_debug_builds_urls))
revision_pattern = revisions.revision_pattern_from_build_bucket_path(
release_build_bucket_path)
found_revision = False
for release_build_url in release_build_urls:
match = re.match(revision_pattern, release_build_url)
if not match:
continue
revision = revisions.convert_revision_to_integer(match.group(1))
if (not other_build_url_lists or all(
revisions.find_build_url(l[0], l[1], revision)
for l in other_build_url_lists)):
found_revision = True
break
if not found_revision:
logs.log_error('Unable to find a matching revision.')
return None
return setup_regular_build(revision)
def execute_task(fuzzer_name_and_revision, job_type):
"""Execute corpus pruning task."""
# TODO(ochang): Remove this once remaining jobs in queue are all processed.
if '@' in fuzzer_name_and_revision:
full_fuzzer_name, revision = fuzzer_name_and_revision.split('@')
revision = revisions.convert_revision_to_integer(revision)
else:
full_fuzzer_name = fuzzer_name_and_revision
revision = 0
fuzz_target = data_handler.get_fuzz_target(full_fuzzer_name)
task_name = 'corpus_pruning_%s_%s' % (full_fuzzer_name, job_type)
# Get status of last execution.
last_execution_metadata = data_handler.get_task_status(task_name)
last_execution_failed = (last_execution_metadata
and last_execution_metadata.status
== data_types.TaskState.ERROR)
# Make sure we're the only instance running for the given fuzzer and
# job_type.
if not data_handler.update_task_status(task_name,
data_types.TaskState.STARTED):
logs.log('A previous corpus pruning task is still running, exiting.')
return
# Setup fuzzer and data bundle.
if not setup.update_fuzzer_and_data_bundles(fuzz_target.engine):
raise CorpusPruningException('Failed to set up fuzzer %s.' %
fuzz_target.engine)
use_minijail = environment.get_value('USE_MINIJAIL')
# TODO(unassigned): Use coverage information for better selection here.
cross_pollinate_fuzzers = _get_cross_pollinate_fuzzers(
fuzz_target.engine, full_fuzzer_name)
context = Context(fuzz_target, cross_pollinate_fuzzers, use_minijail)
# Copy global blacklist into local suppressions file if LSan is enabled.
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# TODO(ochang): Copy this to untrusted worker.
leak_blacklist.copy_global_to_local_blacklist()
try:
result = do_corpus_pruning(context, last_execution_failed, revision)
_save_coverage_information(context, result)
_process_corpus_crashes(context, result)
except CorpusPruningException as e:
logs.log_error('Corpus pruning failed: %s.' % str(e))
data_handler.update_task_status(task_name, data_types.TaskState.ERROR)
return
finally:
context.cleanup()
data_handler.update_task_status(task_name, data_types.TaskState.FINISHED)
def test_convert_revision_to_integer_version_string(self, _):
"""Test version string conversions in convert_revision_to_integer."""
revision = revisions.convert_revision_to_integer('1.1.1.1')
# See the full comment in convert_revision_to_integer, but we pad this with
# zeros to allow sorting.
self.assertEqual(revision, 1000010000100001)
# Ensure that the max lengths for each part are supported.
revision = revisions.convert_revision_to_integer(
'12345.67890.12345.67890')
self.assertEqual(revision, 12345678901234567890)
# Ensure that we raise an exception if any of the individual parts are too
# long.
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('123456.12345.12345.12345')
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('12345.123456.12345.12345')
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('12345.12345.123456.12345')
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('12345.12345.12345.123456')
# Ensure that junk strings also raise value errors.
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('123junk')
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('junk')
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('junk123')
with self.assertRaises(ValueError):
revisions.convert_revision_to_integer('...')
def test_convert_revision_to_integer_simple(self):
"""Test the simple revision case of convert_revision_to_integer."""
revision = revisions.convert_revision_to_integer('12345')
self.assertEqual(revision, 12345)
