def get_corpus(corpus_directory, fuzzer_name):
"""Get corpus directory.
This function will download latest corpus backup file from GCS, unzip
the file and put them in corpus directory.
Args:
directory: The directory to place corpus.
fuzzer_name: Fuzzer name, e.g. libpng_read_fuzzer, xml_parser_fuzzer, etc.
Returns:
True if the corpus can be acquired and False otherwise.
"""
backup_bucket_name = environment.get_value('BACKUP_BUCKET')
corpus_fuzzer_name = environment.get_value('CORPUS_FUZZER_NAME_OVERRIDE')
# Get GCS backup path.
gcs_backup_path = corpus_manager.gcs_url_for_backup_file(
backup_bucket_name, corpus_fuzzer_name, fuzzer_name,
corpus_manager.LATEST_BACKUP_TIMESTAMP)
# Get local backup path.
local_backup_name = os.path.basename(gcs_backup_path)
local_backup_path = os.path.join(corpus_directory, local_backup_name)
# Download latest backup.
if not storage.copy_file_from(gcs_backup_path, local_backup_path):
logs.log_error('Failed to download corpus from GCS bucket {}.'.format(
gcs_backup_path))
return False
# Extract corpus from zip file.
archive.unpack(local_backup_path, corpus_directory)
shell.remove_file(local_backup_path)
return True
def delete(self, remote_path):
"""Delete a remote file."""
fs_path = self.convert_path(remote_path)
shell.remove_file(fs_path)
fs_metadata_path = self.convert_path(remote_path, self.METADATA_DIR)
shell.remove_file(fs_metadata_path)
return True
def clear_old_files(directory, extracted_file_set):
"""Remove files from the directory that isn't in the given file list."""
for root_directory, _, filenames in shell.walk(directory):
for filename in filenames:
file_path = os.path.join(root_directory, filename)
if file_path not in extracted_file_set:
shell.remove_file(file_path)
shell.remove_empty_directories(directory)
def clear_pyc_files(directory):
"""Recursively remove all .pyc files from the given directory"""
for root_directory, _, filenames in shell.walk(directory):
for filename in filenames:
if not filename.endswith('.pyc'):
continue
file_path = os.path.join(root_directory, filename)
shell.remove_file(file_path)
def update_tests_if_needed():
"""Updates layout tests every day."""
data_directory = environment.get_value('FUZZ_DATA')
error_occured = False
expected_task_duration = 60 * 60 # 1 hour.
retry_limit = environment.get_value('FAIL_RETRIES')
temp_archive = os.path.join(data_directory, 'temp.zip')
tests_url = environment.get_value('WEB_TESTS_URL')
# Check if we have a valid tests url.
if not tests_url:
return
# Layout test updates are usually disabled to speedup local testing.
if environment.get_value('LOCAL_DEVELOPMENT'):
return
# |UPDATE_WEB_TESTS| env variable can be used to control our update behavior.
if not environment.get_value('UPDATE_WEB_TESTS'):
return
last_modified_time = persistent_cache.get_value(
TESTS_LAST_UPDATE_KEY, constructor=datetime.datetime.utcfromtimestamp)
if (last_modified_time is not None
and not dates.time_has_expired(last_modified_time,
days=TESTS_UPDATE_INTERVAL_DAYS)):
return
logs.log('Updating layout tests.')
tasks.track_task_start(tasks.Task('update_tests', '', ''),
expected_task_duration)
# Download and unpack the tests archive.
for _ in range(retry_limit):
try:
shell.remove_directory(data_directory, recreate=True)
storage.copy_file_from(tests_url, temp_archive)
archive.unpack(temp_archive, data_directory, trusted=True)
shell.remove_file(temp_archive)
error_occured = False
break
except:
logs.log_error(
'Could not retrieve and unpack layout tests archive. Retrying.'
)
error_occured = True
if not error_occured:
persistent_cache.set_value(TESTS_LAST_UPDATE_KEY,
time.time(),
persist_across_reboots=True)
tasks.track_task_end()
def unpack_testcase(testcase):
"""Unpack a testcase and return all files it is composed of."""
# Figure out where the testcase file should be stored.
input_directory, testcase_file_path = _get_testcase_file_and_path(testcase)
minimized = testcase.minimized_keys and testcase.minimized_keys != 'NA'
if minimized:
key = testcase.minimized_keys
archived = bool(testcase.archive_state
& data_types.ArchiveStatus.MINIMIZED)
else:
key = testcase.fuzzed_keys
archived = bool(testcase.archive_state
& data_types.ArchiveStatus.FUZZED)
if archived:
if minimized:
temp_filename = (os.path.join(
input_directory,
str(testcase.key.id()) + _TESTCASE_ARCHIVE_EXTENSION))
else:
temp_filename = os.path.join(input_directory,
testcase.archive_filename)
else:
temp_filename = testcase_file_path
if not blobs.read_blob_to_disk(key, temp_filename):
return None, input_directory, testcase_file_path
file_list = []
if archived:
archive.unpack(temp_filename, input_directory)
file_list = archive.get_file_list(temp_filename)
shell.remove_file(temp_filename)
file_exists = False
for file_name in file_list:
if os.path.basename(file_name) == os.path.basename(
testcase_file_path):
file_exists = True
break
if not file_exists:
logs.log_error(
'Expected file to run %s is not in archive. Base directory is %s and '
'files in archive are [%s].' %
(testcase_file_path, input_directory, ','.join(file_list)))
return None, input_directory, testcase_file_path
else:
file_list.append(testcase_file_path)
return file_list, input_directory, testcase_file_path
def download_system_symbols_if_needed(symbols_directory):
"""Download system libraries from |SYMBOLS_URL| and cache locally."""
if not should_download_symbols():
return
# Get the build fingerprint parameters.
build_params = settings.get_build_parameters()
if not build_params:
logs.log_error('Unable to determine build parameters.')
return
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
if check_symbols_cached(build_params_check_path, build_params):
return
build_id = build_params.get('build_id')
target = build_params.get('target')
build_type = build_params.get('type')
if not build_id or not target or not build_type:
logs.log_error('Null build parameters found, exiting.')
return
symbols_archive_filename = f'{target}-symbols-{build_id}.zip'
artifact_file_name = symbols_archive_filename
output_filename_override = None
# Include type and sanitizer information in the target.
tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')
target_with_type_and_san = f'{target}-{build_type}'
if tool_suffix and not tool_suffix in target_with_type_and_san:
target_with_type_and_san += f'_{tool_suffix}'
targets_with_type_and_san = [target_with_type_and_san]
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
download_artifact_if_needed(build_id, symbols_directory, symbols_archive_path,
targets_with_type_and_san, artifact_file_name,
output_filename_override)
if not os.path.exists(symbols_archive_path):
logs.log_error(
'Unable to locate symbols archive %s.' % symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
shell.remove_file(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def backup_corpus(backup_bucket_name, corpus, directory):
"""Archive and store corpus as a backup.
Args:
backup_bucket_name: Backup bucket.
corpus: The FuzzTargetCorpus.
directory: Path to directory to be archived and backuped.
Returns:
The backup GCS url, or None on failure.
"""
if not backup_bucket_name:
logs.log('No backup bucket provided, skipping corpus backup.')
return None
dated_backup_url = None
timestamp = str(utils.utcnow().date())
# The archive path for shutil.make_archive should be without an extension.
backup_archive_path = os.path.join(
os.path.dirname(os.path.normpath(directory)), timestamp)
try:
backup_archive_path = shutil.make_archive(backup_archive_path,
BACKUP_ARCHIVE_FORMAT, directory)
dated_backup_url = gcs_url_for_backup_file(
backup_bucket_name, corpus.engine, corpus.project_qualified_target_name,
timestamp)
if not storage.copy_file_to(backup_archive_path, dated_backup_url):
return None
latest_backup_url = gcs_url_for_backup_file(
backup_bucket_name, corpus.engine, corpus.project_qualified_target_name,
LATEST_BACKUP_TIMESTAMP)
if not storage.copy_blob(dated_backup_url, latest_backup_url):
logs.log_error(
'Failed to update latest corpus backup at "%s"' % latest_backup_url)
except Exception as ex:
logs.log_error(
'backup_corpus failed: %s\n' % str(ex),
backup_bucket_name=backup_bucket_name,
directory=directory,
backup_archive_path=backup_archive_path)
finally:
# Remove backup archive.
shell.remove_file(backup_archive_path)
return dated_backup_url
def read_from_disk(testcase_file_path, delete=False):
"""Read the TestcaseRun for the given testcase."""
stats_file_path = TestcaseRun.get_stats_filename(testcase_file_path)
if not os.path.exists(stats_file_path):
return None
fuzzer_run = None
with open(stats_file_path) as f:
fuzzer_run = BaseRun.from_json(f.read())
if delete:
shell.remove_file(stats_file_path)
return fuzzer_run
def generate_new_testcase_mutations_using_radamsa(
corpus_directory, new_testcase_mutations_directory,
generation_timeout):
"""Generate new testcase mutations based on Radamsa."""
radamsa_path = get_radamsa_path()
if not radamsa_path:
# Mutations using radamsa are not supported on current platform, bail out.
return
radamsa_runner = new_process.ProcessRunner(radamsa_path)
files_list = shell.get_files_list(corpus_directory)
filtered_files_list = [
f for f in files_list if os.path.getsize(f) <= CORPUS_INPUT_SIZE_LIMIT
]
if not filtered_files_list:
# No mutations to do on an empty corpus or one with very large files.
return
old_corpus_size = shell.get_directory_file_count(
new_testcase_mutations_directory)
expected_completion_time = time.time() + generation_timeout
for i in range(RADAMSA_MUTATIONS):
original_file_path = random_choice(filtered_files_list)
original_filename = os.path.basename(original_file_path)
output_path = os.path.join(
new_testcase_mutations_directory,
get_radamsa_output_filename(original_filename, i))
result = radamsa_runner.run_and_wait(
['-o', output_path, original_file_path], timeout=RADAMSA_TIMEOUT)
if (os.path.exists(output_path)
and os.path.getsize(output_path) > CORPUS_INPUT_SIZE_LIMIT):
# Skip large files to avoid further mutations and impact fuzzing
# efficiency.
shell.remove_file(output_path)
elif result.return_code or result.timed_out:
logs.log_warn('Radamsa failed to mutate or timed out.',
output=result.output)
# Check if we exceeded our timeout. If yes, do no more mutations and break.
if time.time() > expected_completion_time:
break
new_corpus_size = shell.get_directory_file_count(
new_testcase_mutations_directory)
logs.log('Added %d tests using Radamsa mutations.' %
(new_corpus_size - old_corpus_size))
def _cross_pollinate_other_fuzzer_corpuses(self):
"""Add other fuzzer corpuses to shared corpus path for cross-pollination."""
corpus_backup_date = utils.utcnow().date() - datetime.timedelta(
days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS)
for cross_pollinate_fuzzer in self.cross_pollinate_fuzzers:
project_qualified_name = (
cross_pollinate_fuzzer.fuzz_target.project_qualified_name())
backup_bucket_name = cross_pollinate_fuzzer.backup_bucket_name
corpus_engine_name = cross_pollinate_fuzzer.corpus_engine_name
corpus_backup_url = corpus_manager.gcs_url_for_backup_file(
backup_bucket_name, corpus_engine_name, project_qualified_name,
corpus_backup_date)
corpus_backup_local_filename = '%s-%s' % (
project_qualified_name, os.path.basename(corpus_backup_url))
corpus_backup_local_path = os.path.join(
self.shared_corpus_path, corpus_backup_local_filename)
if not storage.exists(corpus_backup_url, ignore_errors=True):
# This can happen in cases when a new fuzz target is checked in or if
# missed to capture a backup for a particular day (for OSS-Fuzz, this
# will result in a 403 instead of 404 since that GCS path belongs to
# other project). So, just log a warning for debugging purposes only.
logs.log_warn('Corpus backup does not exist, ignoring: %s.' %
corpus_backup_url)
continue
if not storage.copy_file_from(corpus_backup_url,
corpus_backup_local_path):
continue
corpus_backup_output_directory = os.path.join(
self.shared_corpus_path, project_qualified_name)
shell.create_directory(corpus_backup_output_directory)
result = archive.unpack(corpus_backup_local_path,
corpus_backup_output_directory)
shell.remove_file(corpus_backup_local_path)
if result:
logs.log(
'Corpus backup url %s successfully unpacked into shared corpus.'
% corpus_backup_url)
else:
logs.log_error('Failed to unpack corpus backup from url %s.' %
corpus_backup_url)
def remove_testcases_from_directories(directories):
"""Removes all testcases and their dependencies from testcase directories."""
generators = []
for directory in directories:
if not directory.strip():
continue
# If there is a bot-specific files list, delete it now.
bot_testcases_file_path = utils.get_bot_testcases_file_path(directory)
shell.remove_file(bot_testcases_file_path)
generators.append(shell.walk(directory))
for generator in generators:
for structure in generator:
base_directory = structure[0]
for filename in structure[2]:
if not is_testcase_resource(filename):
continue
if filename.startswith(RESOURCES_PREFIX):
# In addition to removing this file, remove all resources.
resources_file_path = os.path.join(base_directory,
filename)
resources = read_resource_list(resources_file_path)
for resource in resources:
shell.remove_file(resource)
file_path = os.path.join(base_directory, filename)
shell.remove_file(file_path)
def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
"""Return true if the data bundle is up to date, false otherwise."""
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
shell.remove_file(sync_file_path)
file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path)
if not os.path.exists(sync_file_path):
return False
last_sync_time = datetime.datetime.utcfromtimestamp(
utils.read_data_from_file(sync_file_path))
# Check if we recently synced.
if not dates.time_has_expired(
last_sync_time, seconds=_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS):
return True
# For search index data bundle, we don't sync them from bucket. Instead, we
# rely on the fuzzer to generate testcases periodically.
if _is_search_index_data_bundle(data_bundle.name):
return False
# Check when the bucket url had last updates. If no new updates, no need to
# update directory.
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
last_updated_time = storage.last_updated(bucket_url)
if last_updated_time and last_sync_time > last_updated_time:
logs.log('Data bundle %s has no new content from last sync.' %
data_bundle.name)
return True
return False
def remove_cache_file_and_metadata(cache_file_path):
"""Removes cache file and its metadata."""
logs.log('Removing cache file %s and its metadata.' % cache_file_path)
shell.remove_file(get_cache_file_metadata_path(cache_file_path))
shell.remove_file(cache_file_path)
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path):
"""Archive testcase and its dependencies, and store in blobstore."""
if not os.path.exists(testcase_path):
logs.log_error('Unable to find testcase %s.' % testcase_path)
return None, None, None, None
absolute_filename = testcase_path
archived = False
zip_filename = None
zip_path = None
if not resource_list:
resource_list = []
# Add resource dependencies based on testcase path. These include
# stuff like extensions directory, dependency files, etc.
resource_list.extend(
testcase_manager.get_resource_dependencies(testcase_path))
# Filter out duplicates, directories, and files that do not exist.
resource_list = utils.filter_file_list(resource_list)
logs.log('Testcase and related files :\n%s' % str(resource_list))
if len(resource_list) <= 1:
# If this does not have any resources, just save the testcase.
# TODO(flowerhack): Update this when we teach CF how to download testcases.
try:
file_handle = open(testcase_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase %s.' % testcase_path)
return None, None, None, None
else:
# If there are resources, create an archive.
# Find the common root directory for all of the resources.
# Assumption: resource_list[0] is the testcase path.
base_directory_list = resource_list[0].split(os.path.sep)
for list_index in range(1, len(resource_list)):
current_directory_list = resource_list[list_index].split(
os.path.sep)
length = min(len(base_directory_list), len(current_directory_list))
for directory_index in range(length):
if (current_directory_list[directory_index] !=
base_directory_list[directory_index]):
base_directory_list = base_directory_list[
0:directory_index]
break
base_directory = os.path.sep.join(base_directory_list)
logs.log('Subresource common base directory: %s' % base_directory)
if base_directory:
# Common parent directory, archive sub-paths only.
base_len = len(base_directory) + len(os.path.sep)
else:
# No common parent directory, archive all paths as it-is.
base_len = 0
# Prepare the filename for the archive.
zip_filename, _ = os.path.splitext(os.path.basename(testcase_path))
zip_filename += _TESTCASE_ARCHIVE_EXTENSION
# Create the archive.
zip_path = os.path.join(environment.get_value('INPUT_DIR'),
zip_filename)
zip_file = zipfile.ZipFile(zip_path, 'w')
for file_name in resource_list:
if os.path.exists(file_name):
relative_filename = file_name[base_len:]
zip_file.write(file_name, relative_filename,
zipfile.ZIP_DEFLATED)
zip_file.close()
try:
file_handle = open(zip_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase archive %s.' % zip_path)
return None, None, None, None
archived = True
absolute_filename = testcase_path[base_len:]
fuzzed_key = blobs.write_blob(file_handle)
file_handle.close()
# Don't need the archive after writing testcase to blobstore.
if zip_path:
shell.remove_file(zip_path)
return fuzzed_key, archived, absolute_filename, zip_filename
def update_fuzzer_and_data_bundles(fuzzer_name):
"""Update the fuzzer with a given name if necessary."""
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == fuzzer_name).get()
if not fuzzer:
logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
raise errors.InvalidFuzzerError
# Set some helper environment variables.
fuzzer_directory = get_fuzzer_directory(fuzzer_name)
environment.set_value('FUZZER_DIR', fuzzer_directory)
environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)
# If the fuzzer generates large testcases or a large number of small ones
# that don't fit on tmpfs, then use the larger disk directory.
if fuzzer.has_large_testcases:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
# Adjust the test timeout, if user has provided one.
if fuzzer.timeout:
environment.set_value('TEST_TIMEOUT', fuzzer.timeout)
# Increase fuzz test timeout if the fuzzer timeout is higher than its
# current value.
fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)
# Adjust the max testcases if this fuzzer has specified a lower limit.
max_testcases = environment.get_value('MAX_TESTCASES')
if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)
# Check for updates to this fuzzer.
version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
if (not fuzzer.builtin
and revisions.needs_update(version_file, fuzzer.revision)):
logs.log('Fuzzer update was found, updating.')
# Clear the old fuzzer directory if it exists.
if not shell.remove_directory(fuzzer_directory, recreate=True):
logs.log_error('Failed to clear fuzzer directory.')
return None
# Copy the archive to local disk and unpack it.
archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
logs.log_error('Failed to copy fuzzer archive.')
return None
try:
archive.unpack(archive_path, fuzzer_directory)
except Exception:
error_message = (
'Failed to unpack fuzzer archive %s '
'(bad archive or unsupported format).') % fuzzer.filename
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return None
fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
if not os.path.exists(fuzzer_path):
error_message = (
'Fuzzer executable %s not found. '
'Check fuzzer configuration.') % fuzzer.executable_path
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return None
# Make fuzzer executable.
os.chmod(fuzzer_path, 0o750)
# Cleanup unneeded archive.
shell.remove_file(archive_path)
# Save the current revision of this fuzzer in a file for later checks.
revisions.write_revision_to_revision_file(version_file,
fuzzer.revision)
logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
_clear_old_data_bundles_if_needed()
# Setup data bundles associated with this fuzzer.
data_bundles = ndb_utils.get_all_from_query(
data_types.DataBundle.query(
data_types.DataBundle.name == fuzzer.data_bundle_name))
for data_bundle in data_bundles:
if not update_data_bundle(fuzzer, data_bundle):
return None
# Setup environment variable for launcher script path.
if fuzzer.launcher_script:
fuzzer_launcher_path = os.path.join(fuzzer_directory,
fuzzer.launcher_script)
environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)
# For launcher script usecase, we need the entire fuzzer directory on the
# worker.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_fuzzer_directory = file_host.rebase_to_worker_root(
fuzzer_directory)
file_host.copy_directory_to_worker(fuzzer_directory,
worker_fuzzer_directory,
replace=True)
return fuzzer
def main():
"""Main sync routine."""
tests_archive_bucket = environment.get_value('TESTS_ARCHIVE_BUCKET')
tests_archive_name = environment.get_value('TESTS_ARCHIVE_NAME')
tests_directory = environment.get_value('TESTS_DIR')
sync_interval = environment.get_value('SYNC_INTERVAL') # in seconds.
shell.create_directory(tests_directory)
# Sync old crash tests.
logs.log('Syncing old crash tests.')
crash_testcases_directory = os.path.join(tests_directory, 'CrashTests')
shell.create_directory(crash_testcases_directory)
unpack_crash_testcases(crash_testcases_directory)
# Sync web tests.
logs.log('Syncing web tests.')
src_directory = os.path.join(tests_directory, 'src')
gclient_file_path = os.path.join(tests_directory, '.gclient')
if not os.path.exists(gclient_file_path):
subprocess.check_call(
['fetch', '--no-history', 'chromium', '--nosvn=True'],
cwd=tests_directory)
if os.path.exists(src_directory):
subprocess.check_call(['gclient', 'revert'], cwd=src_directory)
subprocess.check_call(['git', 'pull'], cwd=src_directory)
subprocess.check_call(['gclient', 'sync'], cwd=src_directory)
else:
raise Exception('Unable to checkout web tests.')
clone_git_repository(tests_directory, 'v8',
'https://chromium.googlesource.com/v8/v8')
clone_git_repository(tests_directory, 'ChakraCore',
'https://github.com/Microsoft/ChakraCore.git')
clone_git_repository(tests_directory, 'gecko-dev',
'https://github.com/mozilla/gecko-dev.git')
clone_git_repository(tests_directory, 'webgl-conformance-tests',
'https://github.com/KhronosGroup/WebGL.git')
checkout_svn_repository(
tests_directory, 'WebKit/LayoutTests',
'http://svn.webkit.org/repository/webkit/trunk/LayoutTests')
checkout_svn_repository(
tests_directory, 'WebKit/JSTests/stress',
'http://svn.webkit.org/repository/webkit/trunk/JSTests/stress')
checkout_svn_repository(
tests_directory, 'WebKit/JSTests/es6',
'http://svn.webkit.org/repository/webkit/trunk/JSTests/es6')
create_gecko_tests_directory(tests_directory, 'gecko-dev', 'gecko-tests')
# Upload tests archive to google cloud storage.
logs.log('Uploading tests archive to cloud.')
tests_archive_local = os.path.join(tests_directory, tests_archive_name)
tests_archive_remote = 'gs://{bucket_name}/{archive_name}'.format(
bucket_name=tests_archive_bucket, archive_name=tests_archive_name)
shell.remove_file(tests_archive_local)
create_symbolic_link(tests_directory, 'gecko-dev/js/src/tests',
'spidermonkey')
create_symbolic_link(tests_directory, 'ChakraCore/test', 'chakra')
# FIXME: Find a way to rename LayoutTests to web_tests without breaking
# compatibility with older testcases.
create_symbolic_link(tests_directory, 'src/third_party/blink/web_tests',
'LayoutTests')
subprocess.check_call(
[
'zip',
'-r',
tests_archive_local,
'CrashTests',
'LayoutTests',
'WebKit',
'gecko-tests',
'v8/test/mjsunit',
'spidermonkey',
'chakra',
'webgl-conformance-tests',
'-x',
'*.cc',
'-x',
'*.cpp',
'-x',
'*.py',
'-x',
'*.txt',
'-x',
'*-expected.*',
'-x',
'*.git*',
'-x',
'*.svn*',
],
cwd=tests_directory)
subprocess.check_call(
['gsutil', 'cp', tests_archive_local, tests_archive_remote])
logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval)
time.sleep(sync_interval)