def test_rsync(self):
"""Tests that rsync works as intended."""
with mock.patch(
'common.gsutil.gsutil_command') as mocked_gsutil_command:
gsutil.rsync(self.SRC, self.DST)
mocked_gsutil_command.assert_called_with(
['rsync', '-d', '-r', '/src', 'gs://dst'])
def test_options(self):
"""Tests that rsync works as intended when supplied a gsutil_options
argument."""
flag = '-flag'
with mock.patch(
'common.gsutil.gsutil_command') as mocked_gsutil_command:
gsutil.rsync(self.SRC, self.DST, options=[flag])
assert flag in mocked_gsutil_command.call_args_list[0][0][0]
def test_no_flag(self, kwarg_for_rsync, flag):
"""Tests that rsync works as intended when caller specifies not
to use specific flags."""
kwargs_for_rsync = {}
kwargs_for_rsync[kwarg_for_rsync] = False
with mock.patch(
'common.gsutil.gsutil_command') as mocked_gsutil_command:
gsutil.rsync(self.SRC, self.DST, **kwargs_for_rsync)
assert flag not in mocked_gsutil_command.call_args_list[0][0][0]
def save_results(self):
"""Save the results directory to GCS."""
if not self.gcs_sync_dir:
return
# Copy results directory before rsyncing it so that we don't get an
# exception from uploading a file that changes in size. Files can change
# in size because the log file containing the fuzzer's output is in this
# directory and can be written to by the fuzzer at any time.
results_copy = filesystem.make_dir_copy(self.results_dir)
gsutil.rsync(results_copy,
posixpath.join(self.gcs_sync_dir, self.results_dir))
def output_report(web_bucket):
"""Generate the HTML report and write it to |web_bucket|."""
experiment_name = experiment_utils.get_experiment_name()
reports_dir = get_reports_dir()
try:
logger.debug('Generating report.')
filesystem.recreate_directory(reports_dir)
generate_report.generate_report([experiment_name], str(reports_dir))
gsutil.rsync(str(reports_dir),
web_bucket,
gsutil_options=[
'-h', 'Cache-Control:public,max-age=0,no-transform'
],
parallel=False)
logger.debug('Done generating report.')
except Exception: # pylint: disable=broad-except
logger.error('Error generating HTML report.')
def copy_resources_to_bucket(config_dir: str, config: Dict):
"""Copy resources the dispatcher will need for the experiment to the
cloud_experiment_bucket."""
cloud_experiment_path = os.path.join(config['cloud_experiment_bucket'],
config['experiment'])
base_destination = os.path.join(cloud_experiment_path, 'input')
# Send the local source repository to the cloud for use by dispatcher.
# Local changes to any file will propagate.
# Filter out unnecessary directories.
options = [
'-x',
('^\\.git/|^\\.pytype/|^\\.venv/|^.*\\.pyc$|^__pycache__/'
'|.*~$|\\.pytest_cache/|.*/test_data/|^third_party/oss-fuzz/out/'
'|^docs/')
]
destination = os.path.join(base_destination, 'src')
gsutil.rsync(utils.ROOT_DIR, destination, options=options)
# Send config files.
destination = os.path.join(base_destination, 'config')
gsutil.rsync(config_dir, destination)
def copy_resources_to_bucket(config_dir: str, config: Dict):
"""Copy resources the dispatcher will need for the experiment to the
cloud_experiment_bucket."""
def filter_file(tar_info):
"""Filter out unnecessary directories."""
if FILTER_SOURCE_REGEX.match(tar_info.name):
return None
return tar_info
cloud_experiment_path = os.path.join(config['cloud_experiment_bucket'],
config['experiment'])
base_destination = os.path.join(cloud_experiment_path, 'input')
# Send the local source repository to the cloud for use by dispatcher.
# Local changes to any file will propagate.
source_archive = 'src.tar.gz'
with tarfile.open(source_archive, 'w:gz') as tar:
tar.add(utils.ROOT_DIR, arcname='', recursive=True, filter=filter_file)
gsutil.cp(source_archive, base_destination + '/', parallel=True)
os.remove(source_archive)
# Send config files.
destination = os.path.join(base_destination, 'config')
gsutil.rsync(config_dir, destination, parallel=True)
def measure_all_trials(experiment: str, max_total_time: int, pool, q) -> bool: # pylint: disable=invalid-name
"""Get coverage data (with coverage runs) for all active trials. Note that
this should not be called unless multiprocessing.set_start_method('spawn')
was called first. Otherwise it will use fork which breaks logging."""
logger.info('Measuring all trials.')
experiment_folders_dir = get_experiment_folders_dir()
if not remote_dir_exists(experiment_folders_dir):
return True
try:
gsutil.rsync(exp_path.gcs(experiment_folders_dir),
str(experiment_folders_dir))
except subprocess.CalledProcessError:
logger.error('Rsyncing experiment folders failed.')
return True
max_cycle = _time_to_cycle(max_total_time)
unmeasured_snapshots = get_unmeasured_snapshots(experiment, max_cycle)
if not unmeasured_snapshots:
return False
measure_trial_coverage_args = [
(unmeasured_snapshot, max_cycle, q)
for unmeasured_snapshot in unmeasured_snapshots
]
result = pool.starmap_async(measure_trial_coverage,
measure_trial_coverage_args)
# Poll the queue for snapshots and save them in batches until the pool is
# done processing each unmeasured snapshot. Then save any remaining
# snapshots.
snapshots = []
snapshots_measured = False
def save_snapshots():
"""Saves measured snapshots if there were any, resets |snapshots| to an
empty list and records the fact that snapshots have been measured."""
if not snapshots:
return
db_utils.bulk_save(snapshots)
snapshots.clear()
nonlocal snapshots_measured
snapshots_measured = True
while True:
try:
snapshot = q.get(timeout=SNAPSHOT_QUEUE_GET_TIMEOUT)
snapshots.append(snapshot)
except queue.Empty:
if result.ready():
# If "ready" that means pool has finished calling on each
# unmeasured_snapshot. Since it is finished and the queue is
# empty, we can stop checking the queue for more snapshots.
break
if len(snapshots) >= SNAPSHOTS_BATCH_SAVE_SIZE * .75:
# Save a smaller batch size if we can make an educated guess
# that we will have to wait for the next snapshot.
save_snapshots()
continue
if len(snapshots) >= SNAPSHOTS_BATCH_SAVE_SIZE and not result.ready():
save_snapshots()
# If we have any snapshots left save them now.
save_snapshots()
return snapshots_measured
