def get_corpus(corpus_directory, fuzzer_name): """Get corpus directory. This function will download latest corpus backup file from GCS, unzip the file and put them in corpus directory. Args: directory: The directory to place corpus. fuzzer_name: Fuzzer name, e.g. libpng_read_fuzzer, xml_parser_fuzzer, etc. Returns: True if the corpus can be acquired and False otherwise. """ backup_bucket_name = environment.get_value('BACKUP_BUCKET') corpus_fuzzer_name = environment.get_value('CORPUS_FUZZER_NAME_OVERRIDE') # Get GCS backup path. gcs_backup_path = corpus_manager.gcs_url_for_backup_file( backup_bucket_name, corpus_fuzzer_name, fuzzer_name, corpus_manager.LATEST_BACKUP_TIMESTAMP) # Get local backup path. local_backup_name = os.path.basename(gcs_backup_path) local_backup_path = os.path.join(corpus_directory, local_backup_name) # Download latest backup. if not storage.copy_file_from(gcs_backup_path, local_backup_path): logs.log_error('Failed to download corpus from GCS bucket {}.'.format( gcs_backup_path)) return False # Extract corpus from zip file. archive.unpack(local_backup_path, corpus_directory) shell.remove_file(local_backup_path) return True
def delete(self, remote_path): """Delete a remote file.""" fs_path = self.convert_path(remote_path) shell.remove_file(fs_path) fs_metadata_path = self.convert_path(remote_path, self.METADATA_DIR) shell.remove_file(fs_metadata_path) return True
def clear_old_files(directory, extracted_file_set): """Remove files from the directory that isn't in the given file list.""" for root_directory, _, filenames in shell.walk(directory): for filename in filenames: file_path = os.path.join(root_directory, filename) if file_path not in extracted_file_set: shell.remove_file(file_path) shell.remove_empty_directories(directory)
def clear_pyc_files(directory): """Recursively remove all .pyc files from the given directory""" for root_directory, _, filenames in shell.walk(directory): for filename in filenames: if not filename.endswith('.pyc'): continue file_path = os.path.join(root_directory, filename) shell.remove_file(file_path)
def update_tests_if_needed(): """Updates layout tests every day.""" data_directory = environment.get_value('FUZZ_DATA') error_occured = False expected_task_duration = 60 * 60 # 1 hour. retry_limit = environment.get_value('FAIL_RETRIES') temp_archive = os.path.join(data_directory, 'temp.zip') tests_url = environment.get_value('WEB_TESTS_URL') # Check if we have a valid tests url. if not tests_url: return # Layout test updates are usually disabled to speedup local testing. if environment.get_value('LOCAL_DEVELOPMENT'): return # |UPDATE_WEB_TESTS| env variable can be used to control our update behavior. if not environment.get_value('UPDATE_WEB_TESTS'): return last_modified_time = persistent_cache.get_value( TESTS_LAST_UPDATE_KEY, constructor=datetime.datetime.utcfromtimestamp) if (last_modified_time is not None and not dates.time_has_expired(last_modified_time, days=TESTS_UPDATE_INTERVAL_DAYS)): return logs.log('Updating layout tests.') tasks.track_task_start(tasks.Task('update_tests', '', ''), expected_task_duration) # Download and unpack the tests archive. for _ in range(retry_limit): try: shell.remove_directory(data_directory, recreate=True) storage.copy_file_from(tests_url, temp_archive) archive.unpack(temp_archive, data_directory, trusted=True) shell.remove_file(temp_archive) error_occured = False break except: logs.log_error( 'Could not retrieve and unpack layout tests archive. Retrying.' ) error_occured = True if not error_occured: persistent_cache.set_value(TESTS_LAST_UPDATE_KEY, time.time(), persist_across_reboots=True) tasks.track_task_end()
def unpack_testcase(testcase): """Unpack a testcase and return all files it is composed of.""" # Figure out where the testcase file should be stored. input_directory, testcase_file_path = _get_testcase_file_and_path(testcase) minimized = testcase.minimized_keys and testcase.minimized_keys != 'NA' if minimized: key = testcase.minimized_keys archived = bool(testcase.archive_state & data_types.ArchiveStatus.MINIMIZED) else: key = testcase.fuzzed_keys archived = bool(testcase.archive_state & data_types.ArchiveStatus.FUZZED) if archived: if minimized: temp_filename = (os.path.join( input_directory, str(testcase.key.id()) + _TESTCASE_ARCHIVE_EXTENSION)) else: temp_filename = os.path.join(input_directory, testcase.archive_filename) else: temp_filename = testcase_file_path if not blobs.read_blob_to_disk(key, temp_filename): return None, input_directory, testcase_file_path file_list = [] if archived: archive.unpack(temp_filename, input_directory) file_list = archive.get_file_list(temp_filename) shell.remove_file(temp_filename) file_exists = False for file_name in file_list: if os.path.basename(file_name) == os.path.basename( testcase_file_path): file_exists = True break if not file_exists: logs.log_error( 'Expected file to run %s is not in archive. Base directory is %s and ' 'files in archive are [%s].' % (testcase_file_path, input_directory, ','.join(file_list))) return None, input_directory, testcase_file_path else: file_list.append(testcase_file_path) return file_list, input_directory, testcase_file_path
def download_system_symbols_if_needed(symbols_directory): """Download system libraries from |SYMBOLS_URL| and cache locally.""" if not should_download_symbols(): return # Get the build fingerprint parameters. build_params = settings.get_build_parameters() if not build_params: logs.log_error('Unable to determine build parameters.') return build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') if check_symbols_cached(build_params_check_path, build_params): return build_id = build_params.get('build_id') target = build_params.get('target') build_type = build_params.get('type') if not build_id or not target or not build_type: logs.log_error('Null build parameters found, exiting.') return symbols_archive_filename = f'{target}-symbols-{build_id}.zip' artifact_file_name = symbols_archive_filename output_filename_override = None # Include type and sanitizer information in the target. tool_suffix = environment.get_value('SANITIZER_TOOL_NAME') target_with_type_and_san = f'{target}-{build_type}' if tool_suffix and not tool_suffix in target_with_type_and_san: target_with_type_and_san += f'_{tool_suffix}' targets_with_type_and_san = [target_with_type_and_san] symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) download_artifact_if_needed(build_id, symbols_directory, symbols_archive_path, targets_with_type_and_san, artifact_file_name, output_filename_override) if not os.path.exists(symbols_archive_path): logs.log_error( 'Unable to locate symbols archive %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) archive.unpack(symbols_archive_path, symbols_directory, trusted=True) shell.remove_file(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)
def backup_corpus(backup_bucket_name, corpus, directory): """Archive and store corpus as a backup. Args: backup_bucket_name: Backup bucket. corpus: The FuzzTargetCorpus. directory: Path to directory to be archived and backuped. Returns: The backup GCS url, or None on failure. """ if not backup_bucket_name: logs.log('No backup bucket provided, skipping corpus backup.') return None dated_backup_url = None timestamp = str(utils.utcnow().date()) # The archive path for shutil.make_archive should be without an extension. backup_archive_path = os.path.join( os.path.dirname(os.path.normpath(directory)), timestamp) try: backup_archive_path = shutil.make_archive(backup_archive_path, BACKUP_ARCHIVE_FORMAT, directory) dated_backup_url = gcs_url_for_backup_file( backup_bucket_name, corpus.engine, corpus.project_qualified_target_name, timestamp) if not storage.copy_file_to(backup_archive_path, dated_backup_url): return None latest_backup_url = gcs_url_for_backup_file( backup_bucket_name, corpus.engine, corpus.project_qualified_target_name, LATEST_BACKUP_TIMESTAMP) if not storage.copy_blob(dated_backup_url, latest_backup_url): logs.log_error( 'Failed to update latest corpus backup at "%s"' % latest_backup_url) except Exception as ex: logs.log_error( 'backup_corpus failed: %s\n' % str(ex), backup_bucket_name=backup_bucket_name, directory=directory, backup_archive_path=backup_archive_path) finally: # Remove backup archive. shell.remove_file(backup_archive_path) return dated_backup_url
def read_from_disk(testcase_file_path, delete=False): """Read the TestcaseRun for the given testcase.""" stats_file_path = TestcaseRun.get_stats_filename(testcase_file_path) if not os.path.exists(stats_file_path): return None fuzzer_run = None with open(stats_file_path) as f: fuzzer_run = BaseRun.from_json(f.read()) if delete: shell.remove_file(stats_file_path) return fuzzer_run
def generate_new_testcase_mutations_using_radamsa( corpus_directory, new_testcase_mutations_directory, generation_timeout): """Generate new testcase mutations based on Radamsa.""" radamsa_path = get_radamsa_path() if not radamsa_path: # Mutations using radamsa are not supported on current platform, bail out. return radamsa_runner = new_process.ProcessRunner(radamsa_path) files_list = shell.get_files_list(corpus_directory) filtered_files_list = [ f for f in files_list if os.path.getsize(f) <= CORPUS_INPUT_SIZE_LIMIT ] if not filtered_files_list: # No mutations to do on an empty corpus or one with very large files. return old_corpus_size = shell.get_directory_file_count( new_testcase_mutations_directory) expected_completion_time = time.time() + generation_timeout for i in range(RADAMSA_MUTATIONS): original_file_path = random_choice(filtered_files_list) original_filename = os.path.basename(original_file_path) output_path = os.path.join( new_testcase_mutations_directory, get_radamsa_output_filename(original_filename, i)) result = radamsa_runner.run_and_wait( ['-o', output_path, original_file_path], timeout=RADAMSA_TIMEOUT) if (os.path.exists(output_path) and os.path.getsize(output_path) > CORPUS_INPUT_SIZE_LIMIT): # Skip large files to avoid further mutations and impact fuzzing # efficiency. shell.remove_file(output_path) elif result.return_code or result.timed_out: logs.log_warn('Radamsa failed to mutate or timed out.', output=result.output) # Check if we exceeded our timeout. If yes, do no more mutations and break. if time.time() > expected_completion_time: break new_corpus_size = shell.get_directory_file_count( new_testcase_mutations_directory) logs.log('Added %d tests using Radamsa mutations.' % (new_corpus_size - old_corpus_size))
def _cross_pollinate_other_fuzzer_corpuses(self): """Add other fuzzer corpuses to shared corpus path for cross-pollination.""" corpus_backup_date = utils.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS) for cross_pollinate_fuzzer in self.cross_pollinate_fuzzers: project_qualified_name = ( cross_pollinate_fuzzer.fuzz_target.project_qualified_name()) backup_bucket_name = cross_pollinate_fuzzer.backup_bucket_name corpus_engine_name = cross_pollinate_fuzzer.corpus_engine_name corpus_backup_url = corpus_manager.gcs_url_for_backup_file( backup_bucket_name, corpus_engine_name, project_qualified_name, corpus_backup_date) corpus_backup_local_filename = '%s-%s' % ( project_qualified_name, os.path.basename(corpus_backup_url)) corpus_backup_local_path = os.path.join( self.shared_corpus_path, corpus_backup_local_filename) if not storage.exists(corpus_backup_url, ignore_errors=True): # This can happen in cases when a new fuzz target is checked in or if # missed to capture a backup for a particular day (for OSS-Fuzz, this # will result in a 403 instead of 404 since that GCS path belongs to # other project). So, just log a warning for debugging purposes only. logs.log_warn('Corpus backup does not exist, ignoring: %s.' % corpus_backup_url) continue if not storage.copy_file_from(corpus_backup_url, corpus_backup_local_path): continue corpus_backup_output_directory = os.path.join( self.shared_corpus_path, project_qualified_name) shell.create_directory(corpus_backup_output_directory) result = archive.unpack(corpus_backup_local_path, corpus_backup_output_directory) shell.remove_file(corpus_backup_local_path) if result: logs.log( 'Corpus backup url %s successfully unpacked into shared corpus.' % corpus_backup_url) else: logs.log_error('Failed to unpack corpus backup from url %s.' % corpus_backup_url)
def remove_testcases_from_directories(directories): """Removes all testcases and their dependencies from testcase directories.""" generators = [] for directory in directories: if not directory.strip(): continue # If there is a bot-specific files list, delete it now. bot_testcases_file_path = utils.get_bot_testcases_file_path(directory) shell.remove_file(bot_testcases_file_path) generators.append(shell.walk(directory)) for generator in generators: for structure in generator: base_directory = structure[0] for filename in structure[2]: if not is_testcase_resource(filename): continue if filename.startswith(RESOURCES_PREFIX): # In addition to removing this file, remove all resources. resources_file_path = os.path.join(base_directory, filename) resources = read_resource_list(resources_file_path) for resource in resources: shell.remove_file(resource) file_path = os.path.join(base_directory, filename) shell.remove_file(file_path)
def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): """Return true if the data bundle is up to date, false otherwise.""" sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) if environment.is_trusted_host() and data_bundle.sync_to_worker: from clusterfuzz._internal.bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) shell.remove_file(sync_file_path) file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path) if not os.path.exists(sync_file_path): return False last_sync_time = datetime.datetime.utcfromtimestamp( utils.read_data_from_file(sync_file_path)) # Check if we recently synced. if not dates.time_has_expired( last_sync_time, seconds=_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS): return True # For search index data bundle, we don't sync them from bucket. Instead, we # rely on the fuzzer to generate testcases periodically. if _is_search_index_data_bundle(data_bundle.name): return False # Check when the bucket url had last updates. If no new updates, no need to # update directory. bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) last_updated_time = storage.last_updated(bucket_url) if last_updated_time and last_sync_time > last_updated_time: logs.log('Data bundle %s has no new content from last sync.' % data_bundle.name) return True return False
def remove_cache_file_and_metadata(cache_file_path): """Removes cache file and its metadata.""" logs.log('Removing cache file %s and its metadata.' % cache_file_path) shell.remove_file(get_cache_file_metadata_path(cache_file_path)) shell.remove_file(cache_file_path)
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path): """Archive testcase and its dependencies, and store in blobstore.""" if not os.path.exists(testcase_path): logs.log_error('Unable to find testcase %s.' % testcase_path) return None, None, None, None absolute_filename = testcase_path archived = False zip_filename = None zip_path = None if not resource_list: resource_list = [] # Add resource dependencies based on testcase path. These include # stuff like extensions directory, dependency files, etc. resource_list.extend( testcase_manager.get_resource_dependencies(testcase_path)) # Filter out duplicates, directories, and files that do not exist. resource_list = utils.filter_file_list(resource_list) logs.log('Testcase and related files :\n%s' % str(resource_list)) if len(resource_list) <= 1: # If this does not have any resources, just save the testcase. # TODO(flowerhack): Update this when we teach CF how to download testcases. try: file_handle = open(testcase_path, 'rb') except IOError: logs.log_error('Unable to open testcase %s.' % testcase_path) return None, None, None, None else: # If there are resources, create an archive. # Find the common root directory for all of the resources. # Assumption: resource_list[0] is the testcase path. base_directory_list = resource_list[0].split(os.path.sep) for list_index in range(1, len(resource_list)): current_directory_list = resource_list[list_index].split( os.path.sep) length = min(len(base_directory_list), len(current_directory_list)) for directory_index in range(length): if (current_directory_list[directory_index] != base_directory_list[directory_index]): base_directory_list = base_directory_list[ 0:directory_index] break base_directory = os.path.sep.join(base_directory_list) logs.log('Subresource common base directory: %s' % base_directory) if base_directory: # Common parent directory, archive sub-paths only. base_len = len(base_directory) + len(os.path.sep) else: # No common parent directory, archive all paths as it-is. base_len = 0 # Prepare the filename for the archive. zip_filename, _ = os.path.splitext(os.path.basename(testcase_path)) zip_filename += _TESTCASE_ARCHIVE_EXTENSION # Create the archive. zip_path = os.path.join(environment.get_value('INPUT_DIR'), zip_filename) zip_file = zipfile.ZipFile(zip_path, 'w') for file_name in resource_list: if os.path.exists(file_name): relative_filename = file_name[base_len:] zip_file.write(file_name, relative_filename, zipfile.ZIP_DEFLATED) zip_file.close() try: file_handle = open(zip_path, 'rb') except IOError: logs.log_error('Unable to open testcase archive %s.' % zip_path) return None, None, None, None archived = True absolute_filename = testcase_path[base_len:] fuzzed_key = blobs.write_blob(file_handle) file_handle.close() # Don't need the archive after writing testcase to blobstore. if zip_path: shell.remove_file(zip_path) return fuzzed_key, archived, absolute_filename, zip_filename
def update_fuzzer_and_data_bundles(fuzzer_name): """Update the fuzzer with a given name if necessary.""" fuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer: logs.log_error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError # Set some helper environment variables. fuzzer_directory = get_fuzzer_directory(fuzzer_name) environment.set_value('FUZZER_DIR', fuzzer_directory) environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # If the fuzzer generates large testcases or a large number of small ones # that don't fit on tmpfs, then use the larger disk directory. if fuzzer.has_large_testcases: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) # Adjust the test timeout, if user has provided one. if fuzzer.timeout: environment.set_value('TEST_TIMEOUT', fuzzer.timeout) # Increase fuzz test timeout if the fuzzer timeout is higher than its # current value. fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout: environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout) # Adjust the max testcases if this fuzzer has specified a lower limit. max_testcases = environment.get_value('MAX_TESTCASES') if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases: environment.set_value('MAX_TESTCASES', fuzzer.max_testcases) # Check for updates to this fuzzer. version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name) if (not fuzzer.builtin and revisions.needs_update(version_file, fuzzer.revision)): logs.log('Fuzzer update was found, updating.') # Clear the old fuzzer directory if it exists. if not shell.remove_directory(fuzzer_directory, recreate=True): logs.log_error('Failed to clear fuzzer directory.') return None # Copy the archive to local disk and unpack it. archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.log_error('Failed to copy fuzzer archive.') return None try: archive.unpack(archive_path, fuzzer_directory) except Exception: error_message = ( 'Failed to unpack fuzzer archive %s ' '(bad archive or unsupported format).') % fuzzer.filename logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ( 'Fuzzer executable %s not found. ' 'Check fuzzer configuration.') % fuzzer.executable_path logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None # Make fuzzer executable. os.chmod(fuzzer_path, 0o750) # Cleanup unneeded archive. shell.remove_file(archive_path) # Save the current revision of this fuzzer in a file for later checks. revisions.write_revision_to_revision_file(version_file, fuzzer.revision) logs.log('Updated fuzzer to revision %d.' % fuzzer.revision) _clear_old_data_bundles_if_needed() # Setup data bundles associated with this fuzzer. data_bundles = ndb_utils.get_all_from_query( data_types.DataBundle.query( data_types.DataBundle.name == fuzzer.data_bundle_name)) for data_bundle in data_bundles: if not update_data_bundle(fuzzer, data_bundle): return None # Setup environment variable for launcher script path. if fuzzer.launcher_script: fuzzer_launcher_path = os.path.join(fuzzer_directory, fuzzer.launcher_script) environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path) # For launcher script usecase, we need the entire fuzzer directory on the # worker. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host worker_fuzzer_directory = file_host.rebase_to_worker_root( fuzzer_directory) file_host.copy_directory_to_worker(fuzzer_directory, worker_fuzzer_directory, replace=True) return fuzzer
def main(): """Main sync routine.""" tests_archive_bucket = environment.get_value('TESTS_ARCHIVE_BUCKET') tests_archive_name = environment.get_value('TESTS_ARCHIVE_NAME') tests_directory = environment.get_value('TESTS_DIR') sync_interval = environment.get_value('SYNC_INTERVAL') # in seconds. shell.create_directory(tests_directory) # Sync old crash tests. logs.log('Syncing old crash tests.') crash_testcases_directory = os.path.join(tests_directory, 'CrashTests') shell.create_directory(crash_testcases_directory) unpack_crash_testcases(crash_testcases_directory) # Sync web tests. logs.log('Syncing web tests.') src_directory = os.path.join(tests_directory, 'src') gclient_file_path = os.path.join(tests_directory, '.gclient') if not os.path.exists(gclient_file_path): subprocess.check_call( ['fetch', '--no-history', 'chromium', '--nosvn=True'], cwd=tests_directory) if os.path.exists(src_directory): subprocess.check_call(['gclient', 'revert'], cwd=src_directory) subprocess.check_call(['git', 'pull'], cwd=src_directory) subprocess.check_call(['gclient', 'sync'], cwd=src_directory) else: raise Exception('Unable to checkout web tests.') clone_git_repository(tests_directory, 'v8', 'https://chromium.googlesource.com/v8/v8') clone_git_repository(tests_directory, 'ChakraCore', 'https://github.com/Microsoft/ChakraCore.git') clone_git_repository(tests_directory, 'gecko-dev', 'https://github.com/mozilla/gecko-dev.git') clone_git_repository(tests_directory, 'webgl-conformance-tests', 'https://github.com/KhronosGroup/WebGL.git') checkout_svn_repository( tests_directory, 'WebKit/LayoutTests', 'http://svn.webkit.org/repository/webkit/trunk/LayoutTests') checkout_svn_repository( tests_directory, 'WebKit/JSTests/stress', 'http://svn.webkit.org/repository/webkit/trunk/JSTests/stress') checkout_svn_repository( tests_directory, 'WebKit/JSTests/es6', 'http://svn.webkit.org/repository/webkit/trunk/JSTests/es6') create_gecko_tests_directory(tests_directory, 'gecko-dev', 'gecko-tests') # Upload tests archive to google cloud storage. logs.log('Uploading tests archive to cloud.') tests_archive_local = os.path.join(tests_directory, tests_archive_name) tests_archive_remote = 'gs://{bucket_name}/{archive_name}'.format( bucket_name=tests_archive_bucket, archive_name=tests_archive_name) shell.remove_file(tests_archive_local) create_symbolic_link(tests_directory, 'gecko-dev/js/src/tests', 'spidermonkey') create_symbolic_link(tests_directory, 'ChakraCore/test', 'chakra') # FIXME: Find a way to rename LayoutTests to web_tests without breaking # compatibility with older testcases. create_symbolic_link(tests_directory, 'src/third_party/blink/web_tests', 'LayoutTests') subprocess.check_call( [ 'zip', '-r', tests_archive_local, 'CrashTests', 'LayoutTests', 'WebKit', 'gecko-tests', 'v8/test/mjsunit', 'spidermonkey', 'chakra', 'webgl-conformance-tests', '-x', '*.cc', '-x', '*.cpp', '-x', '*.py', '-x', '*.txt', '-x', '*-expected.*', '-x', '*.git*', '-x', '*.svn*', ], cwd=tests_directory) subprocess.check_call( ['gsutil', 'cp', tests_archive_local, tests_archive_remote]) logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval) time.sleep(sync_interval)