def test_sync_projects_update(self):
"""Testing sync_projects() updating a schedule."""
cloud_scheduler_client = CloudSchedulerClient()
with ndb.Client().context():
Project(name='test1',
schedule='0 8 * * *',
project_yaml_contents='',
dockerfile_contents='').put()
Project(name='test2',
schedule='0 9 * * *',
project_yaml_contents='',
dockerfile_contents='').put()
projects = {
'test1': ProjectMetadata('0 8 * * *', '', ''),
'test2': ProjectMetadata('0 7 * * *', '', '')
}
sync_projects(cloud_scheduler_client, projects)
projects_query = Project.query()
self.assertEqual({
'test1': '0 8 * * *',
'test2': '0 7 * * *'
}, {project.name: project.schedule
for project in projects_query})
def test_get_coverage_build_steps(self, mocked_url, mocked_corpora_steps,
mocked_time):
"""Test for get_build_steps."""
del mocked_url, mocked_corpora_steps, mocked_time
datetime.datetime = test_utils.SpoofedDatetime
project_yaml_contents = ('language: c++\n'
'sanitizers:\n'
' - address\n'
'architectures:\n'
' - x86_64\n')
dockerfile_contents = 'test line'
image_project = 'oss-fuzz'
base_images_project = 'oss-fuzz-base'
expected_build_steps_file_path = test_utils.get_test_data_file_path(
'expected_coverage_build_steps.json')
with open(expected_build_steps_file_path) as expected_build_steps_file:
expected_coverage_build_steps = json.load(expected_build_steps_file)
with ndb.Client().context():
Project(name='test-project',
project_yaml_contents=project_yaml_contents,
dockerfile_contents=dockerfile_contents).put()
dockerfile_lines = dockerfile_contents.split('\n')
build_steps = get_build_steps('test-project', project_yaml_contents,
dockerfile_lines, image_project,
base_images_project)
self.assertEqual(build_steps, expected_coverage_build_steps)
def sync_projects(cloud_scheduler_client, projects):
"""Sync projects with cloud datastore."""
for project in Project.query():
if project.name in projects:
continue
logging.info('Deleting project %s', project.name)
try:
delete_scheduler(cloud_scheduler_client, project.name)
project.key.delete()
except exceptions.GoogleAPICallError as error:
logging.error('Scheduler deletion for %s failed with %s',
project.name, error)
existing_projects = {project.name for project in Project.query()}
for project_name in projects:
if project_name in existing_projects:
continue
try:
create_scheduler(cloud_scheduler_client, project_name,
projects[project_name].schedule)
project_metadata = projects[project_name]
Project(
name=project_name,
schedule=project_metadata.schedule,
project_yaml_contents=project_metadata.project_yaml_contents,
dockerfile_contents=project_metadata.dockerfile_contents).put(
)
except exceptions.GoogleAPICallError as error:
logging.error('Scheduler creation for %s failed with %s',
project_name, error)
for project in Project.query():
if project.name not in projects:
continue
logging.info('Setting up project %s', project.name)
project_metadata = projects[project.name]
project_changed = False
if project.schedule != project_metadata.schedule:
try:
logging.info('Schedule changed.')
update_scheduler(cloud_scheduler_client, project,
projects[project.name].schedule)
project.schedule = project_metadata.schedule
project_changed = True
except exceptions.GoogleAPICallError as error:
logging.error('Updating scheduler for %s failed with %s',
project.name, error)
if project.project_yaml_contents != project_metadata.project_yaml_contents:
project.project_yaml_contents = project_metadata.project_yaml_contents
project_changed = True
if project.dockerfile_contents != project_metadata.dockerfile_contents:
project.dockerfile_contents = project_metadata.dockerfile_contents
project_changed = True
if project_changed:
project.put()
def sync_projects(cloud_scheduler_client, projects):
"""Sync projects with cloud datastore."""
for project in Project.query():
if project.name not in projects:
delete_project(cloud_scheduler_client, project)
existing_projects = {project.name for project in Project.query()}
for project_name in projects:
if project_name in existing_projects:
continue
try:
create_scheduler(cloud_scheduler_client, project_name,
projects[project_name].schedule,
build_project.FUZZING_BUILD_TAG,
FUZZING_BUILD_TOPIC)
create_scheduler(cloud_scheduler_client, project_name,
COVERAGE_SCHEDULE,
build_and_run_coverage.COVERAGE_BUILD_TAG,
COVERAGE_BUILD_TOPIC)
project_metadata = projects[project_name]
Project(
name=project_name,
schedule=project_metadata.schedule,
project_yaml_contents=project_metadata.project_yaml_contents,
dockerfile_contents=project_metadata.dockerfile_contents).put(
)
except exceptions.GoogleAPICallError as error:
logging.error('Scheduler creation for %s failed with %s',
project_name, error)
for project in Project.query():
if project.name not in projects:
continue
logging.info('Setting up project %s', project.name)
project_metadata = projects[project.name]
project_changed = False
if project.schedule != project_metadata.schedule:
try:
logging.info('Schedule changed.')
update_scheduler(cloud_scheduler_client, project,
projects[project.name].schedule,
build_project.FUZZING_BUILD_TAG)
project.schedule = project_metadata.schedule
project_changed = True
except exceptions.GoogleAPICallError as error:
logging.error('Updating scheduler for %s failed with %s',
project.name, error)
if project.project_yaml_contents != project_metadata.project_yaml_contents:
project.project_yaml_contents = project_metadata.project_yaml_contents
project_changed = True
if project.dockerfile_contents != project_metadata.dockerfile_contents:
project.dockerfile_contents = project_metadata.dockerfile_contents
project_changed = True
if project_changed:
project.put()
def test_sync_projects_create(self):
""""Testing sync_projects() creating new schedule."""
cloud_scheduler_client = CloudSchedulerClient()
with ndb.Client().context():
Project(name='test1',
schedule='0 8 * * *',
project_yaml_contents='',
dockerfile_contents='').put()
projects = {
'test1': ProjectMetadata('0 8 * * *', '', ''),
'test2': ProjectMetadata('0 7 * * *', '', '')
}
sync_projects(cloud_scheduler_client, projects)
projects_query = Project.query()
self.assertEqual({
'test1': '0 8 * * *',
'test2': '0 7 * * *'
}, {project.name: project.schedule
for project in projects_query})
self.assertCountEqual([
{
'name': 'projects/test-project/location/us-central1/jobs/'
'test2-scheduler-fuzzing',
'pubsub_target': {
'topic_name':
'projects/test-project/topics/request-build',
'data': b'test2'
},
'schedule': '0 7 * * *'
},
{
'name': 'projects/test-project/location/us-central1/jobs/'
'test2-scheduler-coverage',
'pubsub_target': {
'topic_name':
'projects/test-project/topics/request-coverage-build',
'data': b'test2'
},
'schedule': '0 6 * * *'
},
], cloud_scheduler_client.schedulers)
def test_get_build_steps(self, mocked_url, mocked_time):
"""Test for get_build_steps."""
del mocked_url, mocked_time
datetime.datetime = SpoofedDatetime
project_yaml_contents = 'language: c++\nsanitizers:\n - address\narchitectures:\n - x86_64\n'
image_project = 'oss-fuzz'
base_images_project = 'oss-fuzz-base'
testcase_path = os.path.join(os.path.dirname(__file__),
'expected_build_steps.json')
with open(testcase_path) as testcase_file:
expected_build_steps = json.load(testcase_file)
with ndb.Client().context():
Project(name='test-project',
project_yaml_contents=project_yaml_contents,
dockerfile_contents='test line').put()
build_steps = get_build_steps('test-project', image_project,
base_images_project)
self.assertEqual(build_steps, expected_build_steps)