def send(to_email, subject, html_content):
"""Send email."""
sendgrid_api_key = db_config.get_value('sendgrid_api_key')
if not sendgrid_api_key:
logs.log_warn(
'Skipping email as SendGrid API key is not set in config.')
return
from_email = db_config.get_value('sendgrid_sender')
if not from_email:
logs.log_warn(
'Skipping email as SendGrid sender is not set in config.')
return
message = Mail(from_email=From(str(from_email)),
to_emails=To(str(to_email)),
subject=Subject(subject),
html_content=HtmlContent(str(html_content)))
try:
sg = SendGridAPIClient(sendgrid_api_key)
response = sg.send(message)
logs.log('Sent email to %s.' % to_email,
status_code=response.status_code,
body=response.body,
headers=response.headers)
except Exception:
logs.log_error('Failed to send email to %s.' % to_email)
def get_access_token(verification_code):
"""Get the access token from verification code.
See: https://developers.google.com/identity/protocols/OAuth2InstalledApp
"""
client_id = db_config.get_value('reproduce_tool_client_id')
if not client_id:
raise helpers.UnauthorizedException('Client id not configured.')
client_secret = db_config.get_value('reproduce_tool_client_secret')
if not client_secret:
raise helpers.UnauthorizedException('Client secret not configured.')
response = requests.post(
'https://www.googleapis.com/oauth2/v4/token',
headers={'Content-Type': 'application/x-www-form-urlencoded'},
data={
'code': verification_code,
'client_id': client_id,
'client_secret': client_secret,
'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob',
'grant_type': 'authorization_code'
})
if response.status_code != 200:
raise helpers.UnauthorizedException('Invalid verification code (%s): %s' %
(verification_code, response.text))
try:
data = json.loads(response.text)
return data['access_token']
except (KeyError, ValueError):
raise helpers.EarlyExitException(
'Parsing the JSON response body failed: %s' % response.text, 500)
def post(self):
"""Download the reproduce tool configuration json."""
client_id = db_config.get_value('reproduce_tool_client_id')
if not client_id:
return self.render_json({
'error': 'Reproduce tool is not configured.'
}, 500)
domain = data_handler.get_domain()
link_format = 'https://{domain}/{handler}'
configuration = {
'testcase_info_url':
link_format.format(
domain=domain, handler='reproduce-tool/testcase-info'),
'testcase_download_url':
link_format.format(
domain=domain, handler='testcase-detail/download-testcase'),
'oauth_url':
'https://accounts.google.com/o/oauth2/v2/auth?{}'.format(
urllib.parse.urlencode({
'client_id': client_id,
'scope': 'email profile',
'response_type': 'code',
'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob'
})),
}
return self.render_json(configuration)
def get(self):
"""Get the HTML page."""
documentation_url = db_config.get_value('documentation_url')
if not documentation_url:
documentation_url = DEFAULT_DOCUMENTATION_URL
self.redirect(documentation_url)
def get(self):
"""Get the HTML page."""
bug_report_url = db_config.get_value('bug_report_url')
if not bug_report_url:
bug_report_url = DEFAULT_BUG_REPORT_URL
self.redirect(bug_report_url)
def get(self):
"""Process a GET request."""
subscription = db_config.get_value('predator_result_topic')
if not subscription:
logs.log('No Predator subscription configured. Aborting.')
return
client = pubsub.PubSubClient()
messages = client.pull_from_subscription(subscription,
acknowledge=True)
for message in messages:
message = json.loads(message.data)
testcase_id = message['crash_identifiers']
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
logs.log('Testcase %s no longer exists.' % str(testcase_id))
continue
testcase.set_metadata('predator_result',
message,
update_testcase=False)
testcase.delete_metadata('blame_pending', update_testcase=False)
testcase.put()
logs.log('Set predator result for testcase %d.' %
testcase.key.id())
logs.log('Finished processing predator results. %d total.' %
len(messages))
def execute_task(testcase_id, _):
"""Attempt to find the CL introducing the bug associated with testcase_id."""
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
# Make sure that predator topic is configured. If not, nothing to do here.
topic = db_config.get_value('predator_crash_topic')
if not topic:
logs.log('Predator is not configured, skipping blame task.')
return
data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED)
# Prepare pubsub message to send to predator.
message = _prepare_predator_message(testcase)
if not message:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(
testcase, data_types.TaskState.ERROR,
'Failed to generate request for Predator.')
return
# Clear existing results and mark blame result as pending.
testcase = data_handler.get_testcase_by_id(testcase_id)
_clear_blame_result_and_set_pending_flag(testcase)
# Post request to pub sub.
client = pubsub.PubSubClient()
message_ids = client.publish(topic, [message])
logs.log('Successfully published testcase %s to Predator. Message IDs: %s.' %
(testcase_id, message_ids))
data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED)
def _is_blacklisted_user(email):
"""Check if an email is in the privileged users list."""
blacklisted_user_emails = (db_config.get_value('blacklisted_users')
or '').splitlines()
return any(
utils.emails_equal(email, blacklisted_user_email)
for blacklisted_user_email in blacklisted_user_emails)
def get_client():
"""Return client with connection to build apiary."""
# Connect using build apiary service account credentials.
build_apiary_service_account_email = db_config.get_value(
'build_apiary_service_account_email')
build_apiary_service_account_private_key = db_config.get_value(
'build_apiary_service_account_private_key')
credentials = ServiceAccountCredentials.from_p12_keyfile_buffer(
build_apiary_service_account_email,
StringIO.StringIO(build_apiary_service_account_private_key),
scopes='https://www.googleapis.com/auth/androidbuild.internal')
client = apiclient.discovery.build('androidbuildinternal',
'v2beta1',
credentials=credentials,
cache_discovery=False)
return client
def _is_privileged_user(email):
"""Check if an email is in the privileged users list."""
privileged_user_emails = (db_config.get_value('privileged_users') or
'').splitlines()
for privileged_user_email in privileged_user_emails:
if utils.emails_equal(email, privileged_user_email):
return True
return False
def get_email_and_access_token(authorization):
"""Get user email from the request.
See: https://developers.google.com/identity/protocols/OAuth2InstalledApp
"""
if authorization.startswith(VERIFICATION_CODE_PREFIX):
verification_code = authorization.split(' ')[1]
access_token = get_access_token(verification_code)
authorization = BEARER_PREFIX + access_token
if not authorization.startswith(BEARER_PREFIX):
raise helpers.UnauthorizedException(
'The Authorization header is invalid. It should have been started with'
" '%s'." % BEARER_PREFIX)
access_token = authorization.split(' ')[1]
response = requests.get('https://www.googleapis.com/oauth2/v3/tokeninfo',
params={'access_token': access_token})
if response.status_code != 200:
raise helpers.UnauthorizedException(
'Failed to authorize. The Authorization header (%s) might be invalid.'
% authorization)
try:
data = json.loads(response.text)
# Whitelist service accounts. They have different client IDs (or aud).
# Therefore, we check against their email directly.
if data.get('email_verified') and data.get('email') in _auth_config(
).get('whitelisted_oauth_emails', default=[]):
return data['email'], authorization
# Validate that this is an explicitly whitelisted client ID, or the client
# ID for the reproduce tool.
whitelisted_client_ids = _auth_config().get(
'whitelisted_oauth_client_ids', default=[])
reproduce_tool_client_id = db_config.get_value(
'reproduce_tool_client_id')
if reproduce_tool_client_id:
whitelisted_client_ids += [reproduce_tool_client_id]
if data.get('aud') not in whitelisted_client_ids:
raise helpers.UnauthorizedException(
"The access token doesn't belong to one of the allowed OAuth clients"
': %s.' % response.text)
if not data.get('email_verified'):
raise helpers.UnauthorizedException(
'The email (%s) is not verified: %s.' %
(data.get('email'), response.text))
return data['email'], authorization
except (KeyError, ValueError) as e:
raise helpers.EarlyExitException(
'Parsing the JSON response body failed: %s' % response.text,
500) from e
def _is_privileged_user(email):
"""Check if an email is in the privileged users list."""
if local_config.AuthConfig().get('all_users_privileged'):
return True
privileged_user_emails = (db_config.get_value('privileged_users')
or '').splitlines()
return any(
utils.emails_equal(email, privileged_user_email)
for privileged_user_email in privileged_user_emails)
def render_forbidden(self, message):
"""Write HTML response for 403."""
contact_string = db_config.get_value('contact_string')
template_values = {
'message': message,
'user_email': helpers.get_user_email(),
'login_url': users.create_login_url(dest_url=self.request.url),
'switch_account_url': make_switch_account_url(self.request.url),
'logout_url': users.create_logout_url(dest_url=self.request.url),
'contact_string': contact_string,
}
self.render('error-403.html', template_values, 403)
def locked_get(self):
content = db_config.get_value('client_credentials')
if not content:
return None
try:
credentials = Credentials.new_from_json(content)
credentials.set_store(self)
except ValueError:
return None
return credentials
def get_github_url(url):
"""Return contents of URL."""
github_credentials = db_config.get_value('github_credentials')
if not github_credentials:
raise ProjectSetupError('No github credentials.')
client_id, client_secret = github_credentials.strip().split(';')
response = requests.get(url, auth=(client_id, client_secret))
if response.status_code != 200:
logs.log_error(
'Failed to get github url: %s' % url, status_code=response.status_code)
response.raise_for_status()
return json.loads(response.text)
def get_repository_for_component(component):
"""Get the repository based on component."""
default_repository = ''
repository = ''
repository_mappings = db_config.get_value('component_repository_mappings')
for line in repository_mappings.splitlines():
current_component, value = line.split(';', 1)
if current_component == 'default':
default_repository = value
elif current_component == component:
repository = value
return repository or default_repository
def get_client():
"""Return client with connection to build apiary."""
# Connect using build apiary service account credentials.
build_apiary_service_account_email = db_config.get_value(
'build_apiary_service_account_email')
build_apiary_service_account_private_key = db_config.get_value(
'build_apiary_service_account_private_key')
if (not build_apiary_service_account_email
or not build_apiary_service_account_private_key):
logs.log(
'Android build apiary credentials are not set, skip artifact fetch.'
)
return None
credentials = ServiceAccountCredentials.from_p12_keyfile_buffer(
build_apiary_service_account_email,
io.BytesIO(build_apiary_service_account_private_key),
scopes='https://www.googleapis.com/auth/androidbuild.internal')
client = apiclient.discovery.build('androidbuildinternal',
'v2beta1',
credentials=credentials,
cache_discovery=False)
return client
def get_github_url(url):
"""Return contents of URL."""
github_credentials = db_config.get_value('github_credentials')
if not github_credentials:
raise OssFuzzSetupException('No github credentials.')
client_id, client_secret = github_credentials.strip().split(';')
url += '?client_id=%s&client_secret=%s' % (client_id, client_secret)
try:
return json.loads(urllib2.urlopen(url).read())
except urllib2.HTTPError as e:
logs.log_error(
'Failed to get url with code %d and response %s.' % (e.code, e.read()))
raise
def get_user_job_type():
"""Return the job_type that is assigned to the current user. None means one
can access any job type. You might want to invoke get_access(..) with
the job type afterward."""
email = helpers.get_user_email()
privileged_user_emails = (db_config.get_value('privileged_users') or
'').splitlines()
for privileged_user_email in privileged_user_emails:
if ';' in privileged_user_email:
tokens = privileged_user_email.split(';')
privileged_user_real_email = tokens[0]
privileged_user_job_type = tokens[1]
if utils.emails_equal(email, privileged_user_real_email):
return privileged_user_job_type
return None
def render_forbidden(self, message):
"""Write HTML response for 403."""
login_url = make_login_url(dest_url=request.url)
user_email = helpers.get_user_email()
if not user_email:
return self.redirect(login_url)
contact_string = db_config.get_value('contact_string')
template_values = {
'message': message,
'user_email': helpers.get_user_email(),
'login_url': login_url,
'switch_account_url': login_url,
'logout_url': make_logout_url(dest_url=request.url),
'contact_string': contact_string,
}
return self.render('error-403.html', template_values, 403)
def get_github_url(url):
"""Return contents of URL."""
github_credentials = db_config.get_value("github_credentials")
if not github_credentials:
raise ProjectSetupError("No github credentials.")
client_id, client_secret = github_credentials.strip().split(";")
response = requests.get(url,
params={
"client_id": client_id,
"client_secret": client_secret
})
if response.status_code != 200:
logs.log_error("Failed to get github url: %s" % url,
status_code=response.status_code)
response.raise_for_status()
return json.loads(response.text)
def get_github_url(url):
"""Return contents of URL."""
github_credentials = db_config.get_value('github_credentials')
if not github_credentials:
raise OssFuzzSetupException('No github credentials.')
client_id, client_secret = github_credentials.strip().split(';')
response = requests.get(url,
params={
'client_id': client_id,
'client_secret': client_secret
})
if response.status_code != 200:
logs.log_error('Failed to get github url: %s' % url,
status_code=response.status_code)
response.raise_for_status()
return json.loads(response.text)
def _get_current_lock_zone():
"""Get the current zone for locking purposes."""
if environment.get_value('LOCAL_DEVELOPMENT', False):
return 'local'
platform = environment.get_platform_group().lower()
platform_group_mappings = db_config.get_value('platform_group_mappings')
for mapping in platform_group_mappings.splitlines():
if ';' not in mapping:
continue
platform_group, zone = mapping.split(';')
if platform_group.strip() == platform:
return zone
# Default to per-platform seperation.
logs.log_warn('Platform group mapping not set in admin configuration, '
'using default platform - %s.' % platform)
return platform
def update_environment_for_job(environment_string):
"""Process the environment variable string included with a job."""
# Stacktraces to ignore for found crashes.
# This is set in admin configuration.
environment.set_value('CRASH_EXCLUSIONS',
db_config.get_value('stack_blacklist'))
# Now parse the job's environment definition.
environment_values = (
environment.parse_environment_definition(environment_string))
for key, value in six.iteritems(environment_values):
environment.set_value(key, value)
# If we share the build with another job type, force us to be a custom binary
# job type.
if environment.get_value('SHARE_BUILD_WITH_JOB_TYPE'):
environment.set_value('CUSTOM_BINARY', True)
if environment.is_trusted_host():
environment_values['JOB_NAME'] = environment.get_value('JOB_NAME')
from bot.untrusted_runner import environment as worker_environment
worker_environment.update_environment(environment_values)
