def mktempfile():
with tempfile.NamedTemporaryFile(delete=False) as f:
name = f.name
try:
yield name
finally:
try:
os.unlink(name)
except OSError as e:
log.debug("No file {0}".format(name))
def find_violations(contents, terms):
"""Find any violations in a given file."""
hits = []
for name, pattern in terms.items():
match = re.search(pattern, contents, flags=re.MULTILINE | re.DOTALL)
if match:
log.debug("Contents hit on pattern {}".format(pattern))
hits.append(name)
return hits
def github_handler(event, context):
"""
Handles the processing of Github commit events.
The general flow of processing is as follows::
1) Receive Github Webhook event.
2) Validate event for SourceIp, User-Agent and HMAC digest using a pre-shared secret.
3) Fetch terms from Scumblr for processing.
4) Fetch commit information from Github.
5) Fetch full file information via the blob api.
6) Analyze blob with terms defined by the Scumblr configuration.
7) Return analysis results to Scumblr.
"""
log.debug('Entering lambda handler with event: {}'.format(
json.dumps(event, indent=2)))
# github has a very low timeout (10s) we make sure that we can prewarm our function to prevent
# the service from timing out
if event.get('source') == 'aws.events':
return {'statusCode': '200', 'body': '{}'}
github.validate(event)
body = json.loads(event['body'])
commit_url = body['repository']['commits_url'][:-len('{/sha}')]
blobs_url = body['repository']['blobs_url'][:-len('{/sha}')]
# get search terms from scumblr
config = scumblr.get_config('GithubEventAnalyzer')
log.debug('Body contains {} commits'.format(len(body['commits'])))
for c in body['commits']:
commit_data = github.request(commit_url + '/' + c['id'])
for f in commit_data['files']:
data = github.request(blobs_url + '/' + f['sha'])['content']
try:
commit_data['contents'] = base64.b64decode(data).decode(
'utf-8', 'ignore')
commit_data['contents_url'] = f['contents_url']
commit_data['committer'] = c['committer']
commit_data['ref'] = body['ref']
commit_data['html_url'] = body['repository']['html_url']
except Exception as e:
log.exception(e)
continue
process_task_configs(commit_data, config)
return {'statusCode': '200', 'body': '{}'}
def request(url, data=None):
"""Attempt to make a scumblr request."""
with mktempfile() as tmpfile:
with open(tmpfile, 'w') as f:
f.write(get_secret("ENCRYPTED_SCUMBLR_KEY").decode('utf-8'))
if data:
data = json.dumps(data, indent=2)
log.debug("Scumblr Request. URL: {0} Data: {1}".format(url, data))
response = requests.post(SCUMBLR_URL + url,
cert=(SCUMBLR_CLIENT_PATH, tmpfile),
data=data)
else:
log.debug("Scumblr Request. URL: {0}".format(url))
response = requests.get(SCUMBLR_URL + url,
cert=(SCUMBLR_CLIENT_PATH, tmpfile))
if not response.ok:
log.debug(response.content)
raise GeneralFailure(
"Request to Scumblr failed. URL: {0} Data: {1}".format(url, data))
log.debug("Scumblr Response. Status: {0}".format(response.status_code, ))
if not data:
return response.json()
def rocketci_handler(event, context):
"""
Handles processing of RocketCI commit events.
The general flow of processing is as follows::
1) Receive RocketCI event.
2) Fetch terms from Scumblr for processing.
3) Fetch commit information from Stash/Bitbucket.
4) Fetch full file information via api.
5) Analyze blob with terms defined by the Scumblr configuration.
6) Return analysis results to Scumblr.
:param event:
:param context:
:return:
"""
log.debug('Entering lambda handler with event: {}'.format(
json.dumps(event, indent=2)))
for r in event['Records']:
body = json.loads(r['Sns']['Message'])
if body.get('eventSource') == 'stash-stable':
if body.get('codeEventType') == 'create_commit':
# get search terms from scumblr
log.debug('Got Message: {}'.format(json.dumps(body, indent=2)))
config = scumblr.get_config(
'GithubEventAnalyzer') # TODO separate out terms
commit_data = bitbucket.request(body['source']['url'] + '/' +
'changes')
for f in commit_data['values']:
file_url = bitbucket.get_file_url(
f['links']['self'][0]['href'])
data = bitbucket.reconstruct_contents(
bitbucket.request(file_url))
# normalize commit data
commit_data['contents'] = data
commit_data['contents_url'] = file_url
commit_data['sha'] = body['source']['sha']
commit_data['committer'] = body['source']['author'][
'email']
commit_data['ref'] = body['source']['refId']
commit_data['html_url'] = body['source']['url']
# send to scumblr
process_task_configs(commit_data, config)
return {'statusCode': '200', 'body': '{}'}
def request(url):
"""Attempt to make a stash request."""
user = os.environ['BITBUCKET_USER']
password = get_secret('ENCRYPTED_BITBUCKET_PASSWORD').decode('utf-8')
url = get_rest_url(url)
log.debug('Bitbucket Request. Url: {} User: {}'.format(url, user))
response = requests.get(url, auth=(user, password))
if not response.ok:
raise GeneralFailure(
'Request to Bitbucket failed. URL: {0}'.format(url))
log.debug('Bitbucket Response. Status: {0} Data: {1}'.format(
response.status_code, json.dumps(response.json(), indent=2)))
return response.json()
def authorize(body, headers, source_ip):
"""Ensures that we have a valid github webhook."""
validate_ip(source_ip, GITHUB_CIDR_WHITELIST)
sha_name, signature = headers['X-Hub-Signature'].split('=')
if sha_name != 'sha1':
raise AuthorizationError('Signature algorithm is not SHA1')
message_hmac = hmac.new(get_secret('ENCRYPTED_WEBHOOK_SECRET'),
body.encode('utf-8'), hashlib.sha1)
if not hmac.compare_digest(signature, message_hmac.hexdigest()):
raise AuthorizationError(
'Computed HMAC {} does not match signature {}'.format(
message_hmac.hexdigest(), signature))
log.debug('Computed HMAC {} matches signature {}'.format(
message_hmac.hexdigest(), signature))
def request(url):
"""Attempt to make a Github request."""
params = {'access_token': get_secret('ENCRYPTED_GITHUB_TOKEN')}
log.debug('Github Request. Url: {}'.format(url))
response = requests.get(url, params=params)
if not response.ok:
raise GeneralFailure('Request to Github failed. URL: {0}'.format(url))
if response.headers['X-RateLimit-Remaining'] == 0:
log.info('Throttled by Github. X-RateLimit-Limit: {0}'.format(
response.headers['X-RateLimit-Limit']))
raise ThrottledError()
log.debug('Github Response. Status: {0} Data: {1}'.format(
response.status_code, json.dumps(response.json(), indent=2)))
return response.json()
def validate_ip(source_ip, whitelist):
"""Determine if we are getting a request from a whitelisted ip."""
log.debug("Validating source IP")
for cidr in whitelist:
if ipaddress.IPv4Address(source_ip) in ipaddress.IPv4Network(cidr):
log.debug("{} is in {}".format(source_ip, cidr))
return True
else:
log.debug("{} is NOT in {}".format(source_ip, cidr))
raise AuthorizationError()