def get(self):
"""Get the HTML page."""
key = self.request.get('key')
if not key:
raise helpers.EarlyExitException('No key provided.', 400)
testcase_id = self.request.get('testcase_id')
if testcase_id:
testcase = helpers.get_testcase(testcase_id)
if not access.can_user_access_testcase(testcase):
raise helpers.AccessDeniedException()
if key not in [testcase.fuzzed_keys, testcase.minimized_keys]:
raise helpers.AccessDeniedException()
else:
if not access.has_access():
raise helpers.AccessDeniedException()
if blobs.get_blob_size(key) > MAX_ALLOWED_CONTENT_SIZE:
raise helpers.EarlyExitException(
'Content exceeds max allowed size.', 400)
try:
content = unicode(blobs.read_key(key), errors='replace')
except Exception:
raise helpers.EarlyExitException('Failed to read content.', 400)
line_count = len(content.splitlines())
size = len(content)
title = '%s, %s' % (utils.get_line_count_string(line_count),
utils.get_size_string(size))
self.render('viewer.html', {'content': content, 'title': title})
def get(self, resource=None):
"""Handle a get request with resource."""
testcase = None
testcase_id = request.args.get('testcase_id')
if not testcase_id and not resource:
raise helpers.EarlyExitException('No file requested.', 400)
if testcase_id:
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
raise helpers.EarlyExitException('Invalid testcase.', 400)
if not resource:
if testcase.minimized_keys and testcase.minimized_keys != 'NA':
resource = testcase.minimized_keys
else:
resource = testcase.fuzzed_keys
fuzzer_binary_name = None
if testcase:
fuzzer_binary_name = testcase.get_metadata('fuzzer_binary_name')
resource = str(urllib.parse.unquote(resource))
blob_info = blobs.get_blob_info(resource)
if not blob_info:
raise helpers.EarlyExitException('File does not exist.', 400)
if (testcase and testcase.fuzzed_keys != blob_info.key() and
testcase.minimized_keys != blob_info.key()):
raise helpers.EarlyExitException('Invalid testcase.', 400)
if (utils.is_oss_fuzz() and testcase and
self.check_public_testcase(blob_info, testcase)):
# Public OSS-Fuzz testcase.
return self._send_blob(
blob_info,
testcase.key.id(),
is_minimized=True,
fuzzer_binary_name=fuzzer_binary_name)
is_minimized = testcase and blob_info.key() == testcase.minimized_keys
if access.has_access():
# User has general access.
return self._send_blob(blob_info, testcase_id, is_minimized,
fuzzer_binary_name)
# If this blobstore file is for a testcase, check if the user has access to
# the testcase.
if not testcase:
raise helpers.AccessDeniedException()
if access.can_user_access_testcase(testcase):
return self._send_blob(blob_info, testcase_id, is_minimized,
fuzzer_binary_name)
raise helpers.AccessDeniedException()
def get(self):
"""Handles get request."""
email = helpers.get_user_email()
if not email:
raise helpers.AccessDeniedException()
is_privileged_or_domain_user = access.has_access(
need_privileged_access=False)
if is_privileged_or_domain_user or _is_uploader_allowed(email):
# Privileged, domain and upload users can see all job and fuzzer names.
allowed_jobs = data_handler.get_all_job_type_names()
allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children(
include_parents=True)
else:
# Check if this is an external user with access to certain fuzzers/jobs.
allowed_jobs = external_users.allowed_jobs_for_user(email)
allowed_fuzzers = external_users.allowed_fuzzers_for_user(
email, include_from_jobs=True)
if not allowed_fuzzers and not allowed_jobs:
raise helpers.AccessDeniedException()
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
result, params = get_result()
return self.render(
'upload.html', {
'fieldValues': {
'blackboxFuzzers':
filter_blackbox_fuzzers(allowed_fuzzers),
'jobs':
allowed_jobs,
'libfuzzerTargets':
filter_target_names(allowed_fuzzers, 'libFuzzer'),
'aflTargets':
filter_target_names(allowed_fuzzers, 'afl'),
'honggfuzzTargets':
filter_target_names(allowed_fuzzers, 'honggfuzz'),
'isChromium':
utils.is_chromium(),
'sandboxedJobs':
data_types.INTERNAL_SANDBOXED_JOB_TYPES,
'csrfToken':
form.generate_csrf_token(),
'isExternalUser':
not is_privileged_or_domain_user,
'uploadInfo':
gcs.prepare_blob_upload()._asdict(),
'hasIssueTracker':
has_issue_tracker,
},
'params': params,
'result': result
})
def get(self):
"""Handles get request."""
email = helpers.get_user_email()
if not email:
raise helpers.AccessDeniedException()
is_privileged_or_domain_user = access.has_access(
need_privileged_access=False)
if is_privileged_or_domain_user or _is_uploader_allowed(email):
# Privileged, domain and upload users can see all job and fuzzer names.
allowed_jobs = data_handler.get_all_job_type_names()
allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children(
include_parents=True)
else:
# Check if this is an external user with access to certain fuzzers/jobs.
allowed_jobs = external_users.allowed_jobs_for_user(email)
allowed_fuzzers = external_users.allowed_fuzzers_for_user(
email, include_from_jobs=True)
if not allowed_fuzzers and not allowed_jobs:
raise helpers.AccessDeniedException()
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
result, params = get_result(self)
self.render(
"upload.html",
{
"fieldValues": {
"jobs":
allowed_jobs,
"libfuzzerTargets":
filter_target_names(allowed_fuzzers, "libFuzzer"),
"aflTargets":
filter_target_names(allowed_fuzzers, "afl"),
"isChromium":
utils.is_chromium(),
"sandboxedJobs":
data_types.INTERNAL_SANDBOXED_JOB_TYPES,
"csrfToken":
form.generate_csrf_token(),
"isExternalUser":
not is_privileged_or_domain_user,
"uploadInfo":
gcs.prepare_blob_upload()._asdict(),
"hasIssueTracker":
has_issue_tracker,
},
"params": params,
"result": result,
},
)
def add_scope(query, params, security_field, job_type_field,
fuzzer_name_field):
"""Add scope to the query according to permissions and modify params."""
scope = get_scope()
add_permissions_to_params(scope, params)
subqueries = []
if scope.is_privileged: # The user can access everything.
return
if scope.everything:
everything_query = query.new_subquery()
everything_query.filter(security_field, False)
subqueries.append(everything_query)
if scope.job_types:
job_query = query.new_subquery()
job_query.filter_in(job_type_field, scope.job_types)
subqueries.append(job_query)
if scope.fuzzer_names:
fuzzer_query = query.new_subquery()
fuzzer_query.filter_in(fuzzer_name_field, scope.fuzzer_names)
subqueries.append(fuzzer_query)
if not subqueries: # The user CANNOT access anything.
raise helpers.AccessDeniedException()
query.union(*subqueries)
def post(self):
"""Handles a post request."""
if not helpers.get_user_email():
raise helpers.AccessDeniedException()
result, _ = get_result(self)
self.render_json(result)
def post(self):
"""Handle a post request."""
if not auth.get_current_user():
raise helpers.AccessDeniedException()
project = request.get('project')
fuzz_target = request.get('fuzz_target')
stacktrace = request.get('stacktrace')
state = stack_analyzer.get_crash_data(stacktrace,
symbolize_flag=False,
fuzz_target=fuzz_target,
already_symbolized=True,
detect_ooms_and_hangs=True)
security_flag = crash_analyzer.is_security_issue(
state.crash_stacktrace, state.crash_type, state.crash_address)
if data_handler.find_testcase(project, state.crash_type,
state.crash_state, security_flag):
new_or_duplicate = 'duplicate'
else:
new_or_duplicate = 'new'
return self.render_json({
'result': new_or_duplicate,
'state': state.crash_state,
'type': state.crash_type,
'security': security_flag,
})
def get(self):
"""Handle a GET request."""
# Create a list of externally contributed fuzzers.
user_email = helpers.get_user_email()
if access.has_access():
# User is an internal user of ClusterFuzz (eg: ClusterFuzz developer).
fuzzers_list = (
data_handler.get_all_fuzzer_names_including_children(
include_parents=True))
jobs_list = data_handler.get_all_job_type_names()
else:
# User is an external user of ClusterFuzz (eg: non-Chrome dev who
# submitted a fuzzer or someone with a project in OSS-Fuzz).
fuzzers_list = external_users.allowed_fuzzers_for_user(
user_email, include_from_jobs=True, include_parents=True)
if not fuzzers_list:
# User doesn't actually have access to any fuzzers.
raise helpers.AccessDeniedException()
jobs_list = external_users.allowed_jobs_for_user(user_email)
fuzzers_list.sort()
jobs_list.sort()
result = {
'info': {
'fuzzers': fuzzers_list,
'jobs': jobs_list,
}
}
self.render('fuzzer-stats.html', result)
def wrapper(self, *args, **kwargs):
"""Wrapper."""
if not access.has_access(
need_privileged_access=need_privileged_access):
raise helpers.AccessDeniedException()
return func(self, *args, **kwargs)
def test_forbidden(self):
"""Test forbidden."""
self.mock.check_access_and_get_testcase.side_effect = (
helpers.AccessDeniedException())
self.app = webtest.TestApp(
webapp2.WSGIApplication([('/', CheckTestcaseAccessHandler)]))
resp = self.app.post_json('/', {'testcaseId': '123'}, expect_errors=True)
self.assertEqual(403, resp.status_int)
def get(self, fuzzer_name=None, job_type=None, logs_date=None):
"""Handle a GET request."""
if not fuzzer_name:
raise helpers.EarlyExitException('Fuzzer name cannot be empty.',
400)
if not job_type:
raise helpers.EarlyExitException('Job type cannot be empty.', 400)
if not logs_date:
raise helpers.EarlyExitException('Logs Date cannot be empty.', 400)
if not access.has_access(fuzzer_name=fuzzer_name, job_type=job_type):
raise helpers.AccessDeniedException()
performance_features, date = _get_performance_report_data(
fuzzer_name, job_type, logs_date)
performance_data = performance_features.rows
analyzer = performance_analyzer.LibFuzzerPerformanceAnalyzer()
# It is possible to break the analysis by requesting outdated stats.
try:
total_time = sum(row['actual_duration']
for row in performance_data)
performance_scores, affected_runs_percents, examples = (
analyzer.analyze_stats(performance_data))
except (KeyError, TypeError, ValueError) as e:
logging.error('Exception during performance analysis: %s\n',
str(e))
raise helpers.EarlyExitException(
'Cannot analyze performance for the requested time period.',
404)
# Build performance analysis result.
performance_issues = analyzer.get_issues(performance_scores,
affected_runs_percents,
examples)
performance_report = performance_analyzer.generate_report(
performance_issues, fuzzer_name, job_type)
report = _get_performance_report(fuzzer_name, job_type,
performance_report)
result = {
'info': {
'date': str(date),
'fuzzer_name': report['fuzzer_name'],
'fuzzer_runs': performance_features.total_count,
'job_type': report['job_type'],
'table_data': _build_rows_and_columns(report),
'total_time': str(datetime.timedelta(seconds=total_time)),
}
}
return self.render('performance-report.html', result)
def wrapper(self):
"""Wrapper."""
if not self.is_cron():
raise helpers.AccessDeniedException('You are not a cron.')
result = func(self)
if result is None:
return 'OK'
return result
def test_no_testcase_id(self):
"""Test no testcase id."""
self.mock.check_access_and_get_testcase.side_effect = (
helpers.AccessDeniedException())
self.app = webtest.TestApp(
webapp2.WSGIApplication([('/', CheckTestcaseAccessHandler)]))
resp = self.app.post_json('/', {}, expect_errors=True)
self.assertEqual(400, resp.status_int)
self.assertRegexpMatches(resp.json['message'], '.*not a number.*')
def test_forbidden(self):
"""Test forbidden."""
self.mock.check_access_and_get_testcase.side_effect = (
helpers.AccessDeniedException())
flaskapp = flask.Flask('testflask')
flaskapp.add_url_rule(
'/', view_func=CheckTestcaseAccessHandler.as_view('/'))
self.app = webtest.TestApp(flaskapp)
resp = self.app.post_json('/', {'testcaseId': '123'}, expect_errors=True)
self.assertEqual(403, resp.status_int)
def wrapper(self, *args, **kwargs):
"""Check to see if this handler has a valid CSRF token provided to it."""
token_value = self.request.get('csrf_token')
user = users.get_current_user()
if not user:
raise helpers.AccessDeniedException('Not logged in.')
query = data_types.CSRFToken.query(
data_types.CSRFToken.value == token_value,
data_types.CSRFToken.user_email == user.email())
token = query.get()
if not token:
raise helpers.AccessDeniedException('Invalid CSRF token.')
# Make sure that the token is not expired.
elif token.expiration_time < datetime.datetime.utcnow():
token.key.delete()
raise helpers.AccessDeniedException('Expired CSRF token.')
return func(self, *args, **kwargs)
def get(self):
"""Get the HTML page."""
key = request.get('key')
if not key:
raise helpers.EarlyExitException('No key provided.', 400)
testcase_id = request.get('testcase_id')
if testcase_id:
testcase = helpers.get_testcase(testcase_id)
if not access.can_user_access_testcase(testcase):
raise helpers.AccessDeniedException()
if key not in [testcase.fuzzed_keys, testcase.minimized_keys]:
raise helpers.AccessDeniedException()
else:
if not access.has_access():
raise helpers.AccessDeniedException()
blob_size = blobs.get_blob_size(key)
if blob_size > MAX_ALLOWED_CONTENT_SIZE:
raise helpers.EarlyExitException(
'Content exceeds max allowed size.', 400)
# TODO(mbarbella): Workaround for an issue in the Cloud Storage API. Remove
# once it is fixed properly upstream:
# https://github.com/googleapis/google-cloud-python/issues/6572
if blob_size:
try:
content = blobs.read_key(key).decode('utf-8', errors='replace')
except Exception:
raise helpers.EarlyExitException('Failed to read content.',
400)
else:
content = u''
line_count = len(content.splitlines())
size = len(content)
title = '%s, %s' % (utils.get_line_count_string(line_count),
utils.get_size_string(size))
return self.render('viewer.html', {'content': content, 'title': title})
def test_invalid_testcase_id(self):
"""Test invalid testcase id."""
self.mock.check_access_and_get_testcase.side_effect = (
helpers.AccessDeniedException())
flaskapp = flask.Flask('testflask')
flaskapp.add_url_rule(
'/', view_func=CheckTestcaseAccessHandler.as_view('/'))
self.app = webtest.TestApp(flaskapp)
resp = self.app.post_json('/', {'testcaseId': 'aaa'}, expect_errors=True)
self.assertEqual(400, resp.status_int)
self.assertRegex(resp.json['message'], '.*not a number.*')
def post(self):
"""Mark the testcase as security-related."""
testcase_id = request.get('testcaseId')
security = request.get('security')
severity = request.get('severity')
testcase = helpers.get_testcase(testcase_id)
if not access.has_access(fuzzer_name=testcase.actual_fuzzer_name(),
job_type=testcase.job_type,
need_privileged_access=True):
raise helpers.AccessDeniedException()
mark(testcase, security, severity)
return self.render_json(show.get_testcase_detail(testcase))
def get(self, extra=None):
"""Handle a GET request."""
if not access.has_access():
# User is an external user of ClusterFuzz (eg: non-Chrome dev who
# submitted a fuzzer or someone with a project in OSS-Fuzz).
user_email = helpers.get_user_email()
fuzzers_list = external_users.allowed_fuzzers_for_user(
user_email, include_from_jobs=True, include_parents=True)
if not fuzzers_list:
# User doesn't actually have access to any fuzzers.
raise helpers.AccessDeniedException(
"You don't have access to any fuzzers.")
return self.render('fuzzer-stats.html', {})
def check_access_and_get_testcase(testcase_id):
"""Check the failed attempt count and get the testcase."""
if not helpers.get_user_email():
raise helpers.UnauthorizedException()
if not testcase_id:
raise helpers.EarlyExitException('No test case specified!', 404)
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
raise helpers.EarlyExitException('Invalid test case!', 404)
if not can_user_access_testcase(testcase):
raise helpers.AccessDeniedException()
return testcase
def _check_user_access_and_get_job_filter(self, fuzzer, job):
"""Check whether the current user has access to stats for the fuzzer or job.
Returns a job filter that should be applied to the query."""
access_by_fuzzer_or_job = access.has_access(fuzzer_name=fuzzer,
job_type=job)
if access_by_fuzzer_or_job:
# User has full access to the fuzzer, or the specified job.
# None means no filters => all jobs.
return _get_filter_from_job(job)
if not job:
# Job not specified and user doesn't have full access to the fuzzer. Check
# if the user has any allowed jobs and use that as a filter.
allowed_jobs = external_users.allowed_jobs_for_user(
helpers.get_user_email())
if allowed_jobs:
return allowed_jobs
raise helpers.AccessDeniedException()
def get(self):
"""Handle a GET request."""
project = request.get('project')
if access.has_access():
# User is an internal user of ClusterFuzz (eg: ClusterFuzz developer).
# Show all projects in the list, since this allows user to pick another
# project as needed.
projects_list = data_handler.get_all_project_names()
# Filter fuzzers and job list if a project is provided.
fuzzers_list = (
data_handler.get_all_fuzzer_names_including_children(
include_parents=True, project=project))
jobs_list = data_handler.get_all_job_type_names(project=project)
else:
# User is an external user of ClusterFuzz (eg: non-Chrome dev who
# submitted a fuzzer or someone with a project in OSS-Fuzz).
user_email = helpers.get_user_email()
# TODO(aarya): Filter fuzzer and job if |project| is provided.
fuzzers_list = sorted(
external_users.allowed_fuzzers_for_user(user_email,
include_from_jobs=True,
include_parents=True))
if not fuzzers_list:
# User doesn't actually have access to any fuzzers.
raise helpers.AccessDeniedException(
"You don't have access to any fuzzers.")
jobs_list = sorted(
external_users.allowed_jobs_for_user(user_email))
projects_list = sorted(
{data_handler.get_project_name(job)
for job in jobs_list})
result = {
'projects': projects_list,
'fuzzers': fuzzers_list,
'jobs': jobs_list,
}
return self.render_json(result)
def post(self):
"""Handle a post request."""
if not auth.get_current_user():
raise helpers.AccessDeniedException()
project = request.get('project')
fuzz_target = request.get('fuzz_target')
stacktrace = request.get('stacktrace')
state = stack_analyzer.get_crash_data(
stacktrace,
symbolize_flag=False,
fuzz_target=fuzz_target,
already_symbolized=True,
detect_ooms_and_hangs=True)
security_flag = crash_analyzer.is_security_issue(
state.crash_stacktrace, state.crash_type, state.crash_address)
result = {
'state': state.crash_state,
'type': state.crash_type,
'security': security_flag,
}
duplicate_testcase = data_handler.find_testcase(
project, state.crash_type, state.crash_state, security_flag)
if duplicate_testcase:
result['result'] = 'duplicate'
result['duplicate_id'] = duplicate_testcase.key.id()
bug_id = (
duplicate_testcase.bug_information or
duplicate_testcase.group_bug_information)
if bug_id:
result['bug_id'] = str(bug_id)
else:
result['result'] = 'new'
return self.render_json(result)
def get_results():
"""Return results."""
is_user = access.has_access()
user_email = helpers.get_user_email()
external_jobs = external_users.allowed_jobs_for_user(user_email)
is_external_user = not is_user and external_jobs
if not is_user and not is_external_user:
raise helpers.AccessDeniedException()
if is_user:
projects = _get_all_project_results()
else:
projects = _get_project_results_for_external_user(external_jobs)
results = {
'info': {
'projects': projects,
'is_internal_user': is_user,
},
}
return results
def do_post(self):
"""Upload a testcase."""
testcase_id = self.request.get('testcaseId')
uploaded_file = self.get_upload()
if testcase_id and not uploaded_file:
testcase = helpers.get_testcase(testcase_id)
if not access.can_user_access_testcase(testcase):
raise helpers.AccessDeniedException()
# Use minimized testcase for upload (if available).
key = (testcase.minimized_keys if testcase.minimized_keys
and testcase.minimized_keys != 'NA' else
testcase.fuzzed_keys)
uploaded_file = blobs.get_blob_info(key)
# Extract filename part from blob.
uploaded_file.filename = os.path.basename(
uploaded_file.filename.replace('\\', os.sep))
job_type = self.request.get('job')
if not job_type:
raise helpers.EarlyExitException('Missing job name.', 400)
if (not data_types.Job.VALID_NAME_REGEX.match(job_type)
or not data_types.Job.query(
data_types.Job.name == job_type).get()):
raise helpers.EarlyExitException('Invalid job name.', 400)
fuzzer_name = ''
job_type_lowercase = job_type.lower()
if 'libfuzzer' in job_type_lowercase:
fuzzer_name = 'libFuzzer'
elif 'afl' in job_type_lowercase:
fuzzer_name = 'afl'
target_name = self.request.get('target')
if not fuzzer_name and target_name:
raise helpers.EarlyExitException(
'Target name is not applicable to non-engine jobs (AFL, libFuzzer).',
400)
if fuzzer_name and not target_name:
raise helpers.EarlyExitException(
'Missing target name for engine job (AFL, libFuzzer).', 400)
if (target_name
and not data_types.Fuzzer.VALID_NAME_REGEX.match(target_name)):
raise helpers.EarlyExitException('Invalid target name.', 400)
fully_qualified_fuzzer_name = ''
if fuzzer_name and target_name:
fully_qualified_fuzzer_name, target_name = find_fuzz_target(
fuzzer_name, target_name, job_type)
if not fully_qualified_fuzzer_name:
raise helpers.EarlyExitException('Target does not exist.', 400)
if not access.has_access(need_privileged_access=False,
job_type=job_type,
fuzzer_name=(fully_qualified_fuzzer_name
or fuzzer_name)):
raise helpers.AccessDeniedException()
multiple_testcases = bool(self.request.get('multiple'))
http_flag = bool(self.request.get('http'))
high_end_job = bool(self.request.get('highEnd'))
bug_information = self.request.get('issue')
crash_revision = self.request.get('revision')
timeout = self.request.get('timeout')
retries = self.request.get('retries')
bug_summary_update_flag = bool(self.request.get('updateIssue'))
additional_arguments = self.request.get('args')
app_launch_command = self.request.get('cmd')
platform_id = self.request.get('platform')
testcase_metadata = self.request.get('metadata')
if testcase_metadata:
try:
testcase_metadata = json.loads(testcase_metadata)
if not isinstance(testcase_metadata, dict):
raise helpers.EarlyExitException(
'Metadata is not a JSON object.', 400)
except Exception:
raise helpers.EarlyExitException('Invalid metadata JSON.', 400)
archive_state = 0
bundled = False
file_path_input = ''
email = helpers.get_user_email()
# If we have a AFL or libFuzzer target, use that for arguments.
# Launch command looks like
# python launcher.py {testcase_path} {target_name}
if target_name:
additional_arguments = '%%TESTCASE%% %s' % target_name
# Certain modifications such as app launch command, issue updates are only
# allowed for privileged users.
privileged_user = access.has_access(need_privileged_access=True)
if not privileged_user:
if bug_information or bug_summary_update_flag:
raise helpers.EarlyExitException(
'You are not privileged to update existing issues.', 400)
need_privileged_access = utils.string_is_true(
data_handler.get_value_from_job_definition(
job_type, 'PRIVILEGED_ACCESS'))
if need_privileged_access:
raise helpers.EarlyExitException(
'You are not privileged to run this job type.', 400)
if app_launch_command:
raise helpers.EarlyExitException(
'You are not privileged to run arbitary launch commands.',
400)
if testcase_metadata:
raise helpers.EarlyExitException(
'You are not privileged to set testcase metadata.', 400)
if crash_revision and crash_revision.isdigit():
crash_revision = int(crash_revision)
else:
crash_revision = 0
if bug_information and not bug_information.isdigit():
raise helpers.EarlyExitException('Bug is not a number.', 400)
if not timeout:
timeout = 0
elif not timeout.isdigit() or timeout == '0':
raise helpers.EarlyExitException(
'Testcase timeout must be a number greater than 0.', 400)
else:
timeout = int(timeout)
if timeout > 120:
raise helpers.EarlyExitException(
'Testcase timeout may not be greater than 120 seconds.',
400)
if retries:
if retries.isdigit():
retries = int(retries)
else:
retries = None
if retries is None or retries > MAX_RETRIES:
raise helpers.EarlyExitException(
'Testcase retries must be a number less than %d.' %
MAX_RETRIES, 400)
else:
retries = None
try:
gestures = ast.literal_eval(self.request.get('gestures'))
except:
gestures = []
if not gestures:
gestures = []
job_queue = tasks.queue_for_job(job_type, is_high_end=high_end_job)
if uploaded_file is not None:
filename = ''.join([
x for x in uploaded_file.filename
if x not in ' ;/?:@&=+$,{}|<>()\\'
])
key = str(uploaded_file.key())
if archive.is_archive(filename):
archive_state = data_types.ArchiveStatus.FUZZED
if archive_state:
if multiple_testcases:
if testcase_metadata:
raise helpers.EarlyExitException(
'Testcase metadata not supported with multiple testcases.',
400)
# Create a job to unpack an archive.
metadata = data_types.BundledArchiveMetadata()
metadata.blobstore_key = key
metadata.timeout = timeout
metadata.job_queue = job_queue
metadata.job_type = job_type
metadata.http_flag = http_flag
metadata.archive_filename = filename
metadata.uploader_email = email
metadata.gestures = gestures
metadata.crash_revision = crash_revision
metadata.additional_arguments = additional_arguments
metadata.bug_information = bug_information
metadata.platform_id = platform_id
metadata.app_launch_command = app_launch_command
metadata.fuzzer_name = fuzzer_name
metadata.overridden_fuzzer_name = fully_qualified_fuzzer_name
metadata.fuzzer_binary_name = target_name
metadata.put()
tasks.add_task('unpack',
str(metadata.key.id()),
job_type,
queue=tasks.queue_for_job(job_type))
# Create a testcase metadata object to show the user their upload.
upload_metadata = data_types.TestcaseUploadMetadata()
upload_metadata.timestamp = datetime.datetime.utcnow()
upload_metadata.filename = filename
upload_metadata.blobstore_key = key
upload_metadata.original_blobstore_key = key
upload_metadata.status = 'Pending'
upload_metadata.bundled = True
upload_metadata.uploader_email = email
upload_metadata.retries = retries
upload_metadata.bug_summary_update_flag = bug_summary_update_flag
upload_metadata.put()
helpers.log('Uploaded multiple testcases.',
helpers.VIEW_OPERATION)
self.render_json({'multiple': True})
return
file_path_input = guess_input_file(uploaded_file, filename)
if not file_path_input:
raise helpers.EarlyExitException((
"Unable to detect which file to launch. The main file\'s name "
'must contain either of %s.' % str(RUN_FILE_PATTERNS)),
400)
else:
raise helpers.EarlyExitException('Please select a file to upload.',
400)
testcase_id = data_handler.create_user_uploaded_testcase(
key,
key,
archive_state,
filename,
file_path_input,
timeout,
job_type,
job_queue,
http_flag,
gestures,
additional_arguments,
bug_information,
crash_revision,
email,
platform_id,
app_launch_command,
fuzzer_name,
fully_qualified_fuzzer_name,
target_name,
bundled,
retries,
bug_summary_update_flag,
additional_metadata=testcase_metadata)
testcase = data_handler.get_testcase_by_id(testcase_id)
issue = issue_tracker_utils.get_issue_for_testcase(testcase)
if issue:
report_url = data_handler.TESTCASE_REPORT_URL.format(
domain=data_handler.get_domain(), testcase_id=testcase_id)
comment = ('ClusterFuzz is analyzing your testcase. '
'Developers can follow the progress at %s.' %
report_url)
issue.save(new_comment=comment)
helpers.log('Uploaded testcase %s' % testcase_id,
helpers.VIEW_OPERATION)
self.render_json({'id': '%s' % testcase_id})
def do_post(self):
"""Upload a testcase."""
email = helpers.get_user_email()
testcase_id = request.get('testcaseId')
uploaded_file = self.get_upload()
if testcase_id and not uploaded_file:
testcase = helpers.get_testcase(testcase_id)
if not access.can_user_access_testcase(testcase):
raise helpers.AccessDeniedException()
# Use minimized testcase for upload (if available).
key = (testcase.minimized_keys if testcase.minimized_keys
and testcase.minimized_keys != 'NA' else
testcase.fuzzed_keys)
uploaded_file = blobs.get_blob_info(key)
# Extract filename part from blob.
uploaded_file.filename = os.path.basename(
uploaded_file.filename.replace('\\', os.sep))
job_type = request.get('job')
if not job_type:
raise helpers.EarlyExitException('Missing job name.', 400)
job = data_types.Job.query(data_types.Job.name == job_type).get()
if not job:
raise helpers.EarlyExitException('Invalid job name.', 400)
fuzzer_name = request.get('fuzzer')
job_type_lowercase = job_type.lower()
if 'libfuzzer' in job_type_lowercase:
fuzzer_name = 'libFuzzer'
elif 'afl' in job_type_lowercase:
fuzzer_name = 'afl'
elif 'honggfuzz' in job_type_lowercase:
fuzzer_name = 'honggfuzz'
is_engine_job = fuzzer_name and environment.is_engine_fuzzer_job(
job_type)
target_name = request.get('target')
if not is_engine_job and target_name:
raise helpers.EarlyExitException(
'Target name is not applicable to non-engine jobs (AFL, libFuzzer).',
400)
if is_engine_job and not target_name:
raise helpers.EarlyExitException(
'Missing target name for engine job (AFL, libFuzzer).', 400)
if (target_name
and not data_types.Fuzzer.VALID_NAME_REGEX.match(target_name)):
raise helpers.EarlyExitException('Invalid target name.', 400)
fully_qualified_fuzzer_name = ''
if is_engine_job and target_name:
if job.is_external():
# External jobs don't run and set FuzzTarget entities as part of
# fuzz_task. Set it here instead.
fuzz_target = (data_handler.record_fuzz_target(
fuzzer_name, target_name, job_type))
fully_qualified_fuzzer_name = fuzz_target.fully_qualified_name(
)
target_name = fuzz_target.binary
else:
fully_qualified_fuzzer_name, target_name = find_fuzz_target(
fuzzer_name, target_name, job_type)
if (not access.has_access(need_privileged_access=False,
job_type=job_type,
fuzzer_name=(fully_qualified_fuzzer_name
or fuzzer_name))
and not _is_uploader_allowed(email)):
raise helpers.AccessDeniedException()
multiple_testcases = bool(request.get('multiple'))
http_flag = bool(request.get('http'))
high_end_job = bool(request.get('highEnd'))
bug_information = request.get('issue')
crash_revision = request.get('revision')
timeout = request.get('timeout')
retries = request.get('retries')
bug_summary_update_flag = bool(request.get('updateIssue'))
quiet_flag = bool(request.get('quiet'))
additional_arguments = request.get('args')
app_launch_command = request.get('cmd')
platform_id = request.get('platform')
issue_labels = request.get('issue_labels')
gestures = request.get('gestures') or '[]'
stacktrace = request.get('stacktrace')
crash_data = None
if job.is_external():
if not stacktrace:
raise helpers.EarlyExitException(
'Stacktrace required for external jobs.', 400)
if not crash_revision:
raise helpers.EarlyExitException(
'Revision required for external jobs.', 400)
crash_data = stack_analyzer.get_crash_data(
stacktrace,
fuzz_target=target_name,
symbolize_flag=False,
already_symbolized=True,
detect_ooms_and_hangs=True)
elif stacktrace:
raise helpers.EarlyExitException(
'Should not specify stacktrace for non-external jobs.', 400)
testcase_metadata = request.get('metadata', {})
if testcase_metadata:
try:
testcase_metadata = json.loads(testcase_metadata)
except Exception as e:
raise helpers.EarlyExitException('Invalid metadata JSON.',
400) from e
if not isinstance(testcase_metadata, dict):
raise helpers.EarlyExitException(
'Metadata is not a JSON object.', 400)
if issue_labels:
testcase_metadata['issue_labels'] = issue_labels
try:
gestures = ast.literal_eval(gestures)
except Exception as e:
raise helpers.EarlyExitException('Failed to parse gestures.',
400) from e
archive_state = 0
bundled = False
file_path_input = ''
# Certain modifications such as app launch command, issue updates are only
# allowed for privileged users.
privileged_user = access.has_access(need_privileged_access=True)
if not privileged_user:
if bug_information or bug_summary_update_flag:
raise helpers.EarlyExitException(
'You are not privileged to update existing issues.', 400)
need_privileged_access = utils.string_is_true(
data_handler.get_value_from_job_definition(
job_type, 'PRIVILEGED_ACCESS'))
if need_privileged_access:
raise helpers.EarlyExitException(
'You are not privileged to run this job type.', 400)
if app_launch_command:
raise helpers.EarlyExitException(
'You are not privileged to run arbitrary launch commands.',
400)
if (testcase_metadata
and not _allow_unprivileged_metadata(testcase_metadata)):
raise helpers.EarlyExitException(
'You are not privileged to set testcase metadata.', 400)
if additional_arguments:
raise helpers.EarlyExitException(
'You are not privileged to add command-line arguments.',
400)
if gestures:
raise helpers.EarlyExitException(
'You are not privileged to run arbitrary gestures.', 400)
# TODO(aarya): Remove once AFL is migrated to engine pipeline.
if target_name:
additional_arguments = '%TESTCASE%'
if crash_revision and crash_revision.isdigit():
crash_revision = int(crash_revision)
else:
crash_revision = 0
if bug_information == '0': # Auto-recover from this bad input.
bug_information = None
if bug_information and not bug_information.isdigit():
raise helpers.EarlyExitException('Bug is not a number.', 400)
if not timeout:
timeout = 0
elif not timeout.isdigit() or timeout == '0':
raise helpers.EarlyExitException(
'Testcase timeout must be a number greater than 0.', 400)
else:
timeout = int(timeout)
if timeout > 120:
raise helpers.EarlyExitException(
'Testcase timeout may not be greater than 120 seconds.',
400)
if retries:
if retries.isdigit():
retries = int(retries)
else:
retries = None
if retries is None or retries > MAX_RETRIES:
raise helpers.EarlyExitException(
'Testcase retries must be a number less than %d.' %
MAX_RETRIES, 400)
else:
retries = None
job_queue = tasks.queue_for_job(job_type, is_high_end=high_end_job)
if uploaded_file is not None:
filename = ''.join([
x for x in uploaded_file.filename
if x not in ' ;/?:@&=+$,{}|<>()\\'
])
key = str(uploaded_file.key())
if archive.is_archive(filename):
archive_state = data_types.ArchiveStatus.FUZZED
if archive_state:
if multiple_testcases:
# Create a job to unpack an archive.
metadata = data_types.BundledArchiveMetadata()
metadata.blobstore_key = key
metadata.timeout = timeout
metadata.job_queue = job_queue
metadata.job_type = job_type
metadata.http_flag = http_flag
metadata.archive_filename = filename
metadata.uploader_email = email
metadata.gestures = gestures
metadata.crash_revision = crash_revision
metadata.additional_arguments = additional_arguments
metadata.bug_information = bug_information
metadata.platform_id = platform_id
metadata.app_launch_command = app_launch_command
metadata.fuzzer_name = fuzzer_name
metadata.overridden_fuzzer_name = fully_qualified_fuzzer_name
metadata.fuzzer_binary_name = target_name
metadata.put()
tasks.add_task('unpack',
str(metadata.key.id()),
job_type,
queue=tasks.queue_for_job(job_type))
# Create a testcase metadata object to show the user their upload.
upload_metadata = data_types.TestcaseUploadMetadata()
upload_metadata.timestamp = datetime.datetime.utcnow()
upload_metadata.filename = filename
upload_metadata.blobstore_key = key
upload_metadata.original_blobstore_key = key
upload_metadata.status = 'Pending'
upload_metadata.bundled = True
upload_metadata.uploader_email = email
upload_metadata.retries = retries
upload_metadata.bug_summary_update_flag = bug_summary_update_flag
upload_metadata.quiet_flag = quiet_flag
upload_metadata.additional_metadata_string = json.dumps(
testcase_metadata)
upload_metadata.bug_information = bug_information
upload_metadata.put()
helpers.log('Uploaded multiple testcases.',
helpers.VIEW_OPERATION)
return self.render_json({'multiple': True})
file_path_input = guess_input_file(uploaded_file, filename)
if not file_path_input:
raise helpers.EarlyExitException((
"Unable to detect which file to launch. The main file\'s name "
'must contain either of %s.' % str(RUN_FILE_PATTERNS)),
400)
else:
raise helpers.EarlyExitException('Please select a file to upload.',
400)
testcase_id = data_handler.create_user_uploaded_testcase(
key,
key,
archive_state,
filename,
file_path_input,
timeout,
job,
job_queue,
http_flag,
gestures,
additional_arguments,
bug_information,
crash_revision,
email,
platform_id,
app_launch_command,
fuzzer_name,
fully_qualified_fuzzer_name,
target_name,
bundled,
retries,
bug_summary_update_flag,
quiet_flag,
additional_metadata=testcase_metadata,
crash_data=crash_data)
if not quiet_flag:
testcase = data_handler.get_testcase_by_id(testcase_id)
issue = issue_tracker_utils.get_issue_for_testcase(testcase)
if issue:
report_url = data_handler.TESTCASE_REPORT_URL.format(
domain=data_handler.get_domain(), testcase_id=testcase_id)
comment = ('ClusterFuzz is analyzing your testcase. '
'Developers can follow the progress at %s.' %
report_url)
issue.save(new_comment=comment)
helpers.log('Uploaded testcase %s' % testcase_id,
helpers.VIEW_OPERATION)
return self.render_json({'id': '%s' % testcase_id})
def wrapper(self):
"""Wrapper."""
if not self.is_cron():
raise helpers.AccessDeniedException('You are not a cron.')
return func(self)
def do_post(self):
"""Upload a testcase."""
email = helpers.get_user_email()
testcase_id = self.request.get("testcaseId")
uploaded_file = self.get_upload()
if testcase_id and not uploaded_file:
testcase = helpers.get_testcase(testcase_id)
if not access.can_user_access_testcase(testcase):
raise helpers.AccessDeniedException()
# Use minimized testcase for upload (if available).
key = (testcase.minimized_keys if testcase.minimized_keys
and testcase.minimized_keys != "NA" else
testcase.fuzzed_keys)
uploaded_file = blobs.get_blob_info(key)
# Extract filename part from blob.
uploaded_file.filename = os.path.basename(
uploaded_file.filename.replace("\\", os.sep))
job_type = self.request.get("job")
if not job_type:
raise helpers.EarlyExitException("Missing job name.", 400)
if (not data_types.Job.VALID_NAME_REGEX.match(job_type)
or not data_types.Job.query(
data_types.Job.name == job_type).get()):
raise helpers.EarlyExitException("Invalid job name.", 400)
fuzzer_name = ""
job_type_lowercase = job_type.lower()
if "libfuzzer" in job_type_lowercase:
fuzzer_name = "libFuzzer"
elif "afl" in job_type_lowercase:
fuzzer_name = "afl"
target_name = self.request.get("target")
if not fuzzer_name and target_name:
raise helpers.EarlyExitException(
"Target name is not applicable to non-engine jobs (AFL, libFuzzer).",
400,
)
if fuzzer_name and not target_name:
raise helpers.EarlyExitException(
"Missing target name for engine job (AFL, libFuzzer).", 400)
if target_name and not data_types.Fuzzer.VALID_NAME_REGEX.match(
target_name):
raise helpers.EarlyExitException("Invalid target name.", 400)
fully_qualified_fuzzer_name = ""
if fuzzer_name and target_name:
fully_qualified_fuzzer_name, target_name = find_fuzz_target(
fuzzer_name, target_name, job_type)
if not fully_qualified_fuzzer_name:
raise helpers.EarlyExitException("Target does not exist.", 400)
if not access.has_access(
need_privileged_access=False,
job_type=job_type,
fuzzer_name=(fully_qualified_fuzzer_name or fuzzer_name),
) and not _is_uploader_allowed(email):
raise helpers.AccessDeniedException()
multiple_testcases = bool(self.request.get("multiple"))
http_flag = bool(self.request.get("http"))
high_end_job = bool(self.request.get("highEnd"))
bug_information = self.request.get("issue")
crash_revision = self.request.get("revision")
timeout = self.request.get("timeout")
retries = self.request.get("retries")
bug_summary_update_flag = bool(self.request.get("updateIssue"))
quiet_flag = bool(self.request.get("quiet"))
additional_arguments = self.request.get("args")
app_launch_command = self.request.get("cmd")
platform_id = self.request.get("platform")
issue_labels = self.request.get("issue_labels")
gestures = self.request.get("gestures") or "[]"
testcase_metadata = self.request.get("metadata", {})
if testcase_metadata:
try:
testcase_metadata = json.loads(testcase_metadata)
except Exception:
raise helpers.EarlyExitException("Invalid metadata JSON.", 400)
if not isinstance(testcase_metadata, dict):
raise helpers.EarlyExitException(
"Metadata is not a JSON object.", 400)
if issue_labels:
testcase_metadata["issue_labels"] = issue_labels
try:
gestures = ast.literal_eval(gestures)
except Exception:
raise helpers.EarlyExitException("Failed to parse gestures.", 400)
archive_state = 0
bundled = False
file_path_input = ""
# Certain modifications such as app launch command, issue updates are only
# allowed for privileged users.
privileged_user = access.has_access(need_privileged_access=True)
if not privileged_user:
if bug_information or bug_summary_update_flag:
raise helpers.EarlyExitException(
"You are not privileged to update existing issues.", 400)
need_privileged_access = utils.string_is_true(
data_handler.get_value_from_job_definition(
job_type, "PRIVILEGED_ACCESS"))
if need_privileged_access:
raise helpers.EarlyExitException(
"You are not privileged to run this job type.", 400)
if app_launch_command:
raise helpers.EarlyExitException(
"You are not privileged to run arbitrary launch commands.",
400)
if testcase_metadata and not _allow_unprivileged_metadata(
testcase_metadata):
raise helpers.EarlyExitException(
"You are not privileged to set testcase metadata.", 400)
if additional_arguments:
raise helpers.EarlyExitException(
"You are not privileged to add command-line arguments.",
400)
if gestures:
raise helpers.EarlyExitException(
"You are not privileged to run arbitrary gestures.", 400)
# TODO(aarya): Remove once AFL is migrated to engine pipeline.
if target_name:
additional_arguments = "%TESTCASE%"
if crash_revision and crash_revision.isdigit():
crash_revision = int(crash_revision)
else:
crash_revision = 0
if bug_information == "0": # Auto-recover from this bad input.
bug_information = None
if bug_information and not bug_information.isdigit():
raise helpers.EarlyExitException("Bug is not a number.", 400)
if not timeout:
timeout = 0
elif not timeout.isdigit() or timeout == "0":
raise helpers.EarlyExitException(
"Testcase timeout must be a number greater than 0.", 400)
else:
timeout = int(timeout)
if timeout > 120:
raise helpers.EarlyExitException(
"Testcase timeout may not be greater than 120 seconds.",
400)
if retries:
if retries.isdigit():
retries = int(retries)
else:
retries = None
if retries is None or retries > MAX_RETRIES:
raise helpers.EarlyExitException(
"Testcase retries must be a number less than %d." %
MAX_RETRIES, 400)
else:
retries = None
job_queue = tasks.queue_for_job(job_type, is_high_end=high_end_job)
if uploaded_file is not None:
filename = "".join(x for x in uploaded_file.filename
if x not in " ;/?:@&=+$,{}|<>()\\")
key = str(uploaded_file.key())
if archive.is_archive(filename):
archive_state = data_types.ArchiveStatus.FUZZED
if archive_state:
if multiple_testcases:
# Create a job to unpack an archive.
metadata = data_types.BundledArchiveMetadata()
metadata.blobstore_key = key
metadata.timeout = timeout
metadata.job_queue = job_queue
metadata.job_type = job_type
metadata.http_flag = http_flag
metadata.archive_filename = filename
metadata.uploader_email = email
metadata.gestures = gestures
metadata.crash_revision = crash_revision
metadata.additional_arguments = additional_arguments
metadata.bug_information = bug_information
metadata.platform_id = platform_id
metadata.app_launch_command = app_launch_command
metadata.fuzzer_name = fuzzer_name
metadata.overridden_fuzzer_name = fully_qualified_fuzzer_name
metadata.fuzzer_binary_name = target_name
metadata.put()
tasks.add_task(
"unpack",
str(metadata.key.id()),
job_type,
queue=tasks.queue_for_job(job_type),
)
# Create a testcase metadata object to show the user their upload.
upload_metadata = data_types.TestcaseUploadMetadata()
upload_metadata.timestamp = datetime.datetime.utcnow()
upload_metadata.filename = filename
upload_metadata.blobstore_key = key
upload_metadata.original_blobstore_key = key
upload_metadata.status = "Pending"
upload_metadata.bundled = True
upload_metadata.uploader_email = email
upload_metadata.retries = retries
upload_metadata.bug_summary_update_flag = bug_summary_update_flag
upload_metadata.quiet_flag = quiet_flag
upload_metadata.additional_metadata_string = json.dumps(
testcase_metadata)
upload_metadata.put()
helpers.log("Uploaded multiple testcases.",
helpers.VIEW_OPERATION)
self.render_json({"multiple": True})
return
file_path_input = guess_input_file(uploaded_file, filename)
if not file_path_input:
raise helpers.EarlyExitException(
("Unable to detect which file to launch. The main file's name "
"must contain either of %s." %
str(RUN_FILE_PATTERNS)),
400,
)
else:
raise helpers.EarlyExitException("Please select a file to upload.",
400)
testcase_id = data_handler.create_user_uploaded_testcase(
key,
key,
archive_state,
filename,
file_path_input,
timeout,
job_type,
job_queue,
http_flag,
gestures,
additional_arguments,
bug_information,
crash_revision,
email,
platform_id,
app_launch_command,
fuzzer_name,
fully_qualified_fuzzer_name,
target_name,
bundled,
retries,
bug_summary_update_flag,
quiet_flag,
additional_metadata=testcase_metadata,
)
if not quiet_flag:
testcase = data_handler.get_testcase_by_id(testcase_id)
issue = issue_tracker_utils.get_issue_for_testcase(testcase)
if issue:
report_url = data_handler.TESTCASE_REPORT_URL.format(
domain=data_handler.get_domain(), testcase_id=testcase_id)
comment = ("ClusterFuzz is analyzing your testcase. "
"Developers can follow the progress at %s." %
report_url)
issue.save(new_comment=comment)
helpers.log("Uploaded testcase %s" % testcase_id,
helpers.VIEW_OPERATION)
self.render_json({"id": "%s" % testcase_id})
def get(self):
raise helpers.AccessDeniedException('this_random_message')
def wrapper(self):
"""Wrapper."""
if not users.is_current_user_admin():
raise helpers.AccessDeniedException('Admin access is required.')
return func(self)
