def subProcessTwo():
print("Running sub process 2")
if xray_recorder.is_sampled():
xray_recorder.put_annotation('AnnotationKey', 'AnnotationValue')
xray_recorder.put_metadata('MetadataKey', 'MetatDataValue')
time.sleep(5)
raise NameError('This is a NameError Exception')
def _get_wiki_url(_url, q):
BASE_URL = _url
payload = {}
resp = {"statusCode": 400}
HOT_TOPICS = [
'cholas', 'cheras', 'pandyas', 'pallavas', 'sangam_era', 'kural'
]
if q:
q = q.split('/')[-1]
if not q:
q = random.choice(HOT_TOPICS)
try:
random_sleep()
if _trigger_exception():
xray_recorder.put_annotation("SIMULATED_ERRORS", "True")
xray_recorder.begin_subsegment("BRITTLE_LEGACY_APP")
d = xray_recorder.current_subsegment()
d.put_annotation("MANUALLY_TRIGGERRED_IN_SUBSEGMENT", "True")
xray_recorder.end_subsegment()
raise Exception("RANDOM_ERROR: Simulate Mystique Failure")
r1 = requests.get(f'{BASE_URL}/{q}', params=payload)
xray_recorder.put_metadata('RESPONSE', resp)
resp["statusCode"] = r1.status_code
z = r1.json()
resp["body"] = json.dumps({"message": z["body"]["message"]})
_ddb_put_item(resp)
except Exception as e:
resp["body"] = json.dumps({"message": str(e)})
return resp
def _ddb_put_item(item):
_ddb = boto3.resource('dynamodb')
_ddb_table = _ddb.Table(os.environ.get('DDB_TABLE_NAME'))
item['_id'] = str(uuid.uuid4())
try:
_ddb_table.put_item(Item=item)
except Exception as e:
xray_recorder.put_annotation("DDB_ERROR", "True")
raise
def api():
"Our API logic, to route calls to correct microservices, and return the result"
if request.method == 'POST': #This code deals with API calls, and sends to microservice chain
#Lets update our micro service endpoints / routes.
update_routes()
json_log('Routes: {}'.format(repr(ROUTES)))
# Read the API payload, and pass on to external microservices
messageRequest = request.get_json()
json_log("Host {} received RequestId {}, asking for services: {}".format(socket.gethostname(),messageRequest['RequestId'],','.join(messageRequest['Services'])))
xray_recorder.put_annotation("RequestId", str(messageRequest['RequestId']))
xray_recorder.put_metadata("Services", messageRequest['Services'])
xray_recorder.put_metadata("InputMessage", messageRequest['Message'])
routerResponse = {'Responses':[]}
errors = 0
json_log(messageRequest)
for service in messageRequest['Services']:
# Select a random microservice endpoint, from global ROUTES, updated from DynamoDB
service_endpoint = ""
try:
service_endpoint = random.choice(ROUTES[service])
json_log('RequestId {} needs Service:{} using Endpoint:{}'.format(messageRequest['RequestId'],service,service_endpoint))
req = requests.post(service_endpoint, timeout=10, json={'RequestId':messageRequest['RequestId'],'Message':messageRequest['Message']}).json()
except KeyError as e:
json_log('Request included a service we dont support, Shutting down server','error')
json_log('service-router may need HA config, or to run the latest code from the dashboard','warning')
service_endpoint = "Error"
func = request.environ.get('werkzeug.server.shutdown')
func() #Code isnt working correctly, quit so instance will be replaced.
#Comment out the lines from 'except KeyError' down to this comment, if you dont want the service-router to restart randomly
except Exception as e: #Catch any issues with upstream microservice
#If this code happens, our response will end up being invalid, as we didnt do each transformation
json_log('Error:"{}" connecting to Service:{} Endpoint:{}'.format(e, service, service_endpoint),'exception')
errors += 1
try:
routerResponse['Responses'].append(req)#Add the microservice response to our list of responses
messageRequest['Message']=req['Message']
except Exception as e:
json_log('Bad response:"{}" from Service:{} Endpoint:{}'.format(e,service,service_endpoint),'exception')
errors += 1
#return an http 200, and our API output
xray_recorder.put_metadata("ErrorCount", errors)
xray_recorder.put_metadata("Response", routerResponse)
if errors:
json_log('RequestId {} completed with {} errors'.format(messageRequest['RequestId'], str(errors)))
else:
json_log('RequestId {} completed successfully'.format(messageRequest['RequestId']))
return json.dumps(routerResponse), 200
return 'Service Map:\n {}'.format(repr(ROUTES)), 200
def index():
if request.method == 'POST':
request_string = request.get_json()[
'string'] # expecting {"string": "blahhh"}
xray_recorder.put_annotation('request_body', repr(request.get_json()))
app.logger.info(f"Reverser called with {request_string}")
lambda_metric(
"reverse_service.string_length", # metric
len(request_string), # value
tags=['serverless:reverser'])
response = {'response': request_string.swapcase()}
return response
return {'hello': 'world'}
def _extract_feed_links(url):
logger.info("Fetching URL: %s", url)
headers = {
# Simple scraper blocker counter-measure
'User-Agent':
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36"
}
try:
response = requests.get(url, headers=headers)
except requests.exceptions.MissingSchema:
url = "https://" + url
response = requests.get(url, headers=headers)
if response.status_code != requests.codes.ok:
logger.debug("Server response: %s", response.text)
return None, f"server responded with {response.status_code}"
doc_type = response.headers['content-type'].split(';')[0]
if doc_type == 'text/html':
# Parse page to search for RSS or Atom feeds
soup = BeautifulSoup(response.text, features='lxml')
atom_links = soup.find_all('link',
attrs={
'rel': 'alternate',
'type': 'application/atom+xml'
})
rss_links = soup.find_all('link',
attrs={
'rel': 'alternate',
'type': 'application/rss+xml'
})
links = atom_links + rss_links
# TODO: add feeds to database every time they are found
if not links:
return None, "no feeds found"
elif len(links) == 1:
return (links[0]['href'], ), None
return links, None
elif doc_type in ('application/rss+xml', 'application/atom+xml',
'text/xml'):
return (url, ), None
else:
xray_recorder.put_annotation("contentType",
response.headers['content-type'])
return None, "unrecognized content"
def get_wiki_data(needle='Python_(programming_language)', RENDER_HTML=False):
# resp = {'statusCode': 404, }
pg_info = {'status': False}
try:
# AWS XRay Annotation
xray_recorder.put_annotation("LEGACY_APP_ON_EC2", "BEGIN_PROCESSING")
_wiki = wikipediaapi.Wikipedia('en')
_wiki_page = _wiki.page(str(needle))
if not _wiki_page.exists():
print('Hell N0!')
pg_info[
'ERROR'] = f'No information available for \'{needle}\'. Be the first person to create a wiki page for this topic.'
else:
pg_info['title'] = _wiki_page.title
pg_info['summary'] = _wiki_page.summary[0:100]
pg_info['url'] = _wiki_page.fullurl
pg_info['status'] = True
# AWS XRay Metadata
xray_recorder.put_metadata('WIKI_QUERY_INFO', pg_info)
except Exception as e:
print(str(e))
pg_info['ERROR'] = str(e)
# Deliver as web page using HTML/CSS if NEEDED, set using global variable.
if RENDER_HTML:
return render_template("wiki_page.html",
_wiki_needle=str(needle),
_wiki_page_info=pg_info)
else:
# return jsonify(pg_info)
# return pg_info
# Prep for API Gateway
return {'statusCode': 200, 'body': {'message': pg_info}}
def __enter__(self):
if self.client is None:
self.client = self.ctx.session_factory(assume=False).client('xray')
self.emitter.client = self.client
if self.in_lambda:
self.segment = xray_recorder.begin_subsegment(self.service_name)
else:
self.segment = xray_recorder.begin_segment(
self.service_name, sampling=True)
p = self.ctx.policy
xray_recorder.put_annotation('policy', p.name)
xray_recorder.put_annotation('resource', p.resource_type)
if self.ctx.options.account_id:
xray_recorder.put_annotation('account', self.ctx.options.account_id)
def __enter__(self):
if self.client is None:
self.client = self.ctx.session_factory(assume=False).client('xray')
self.emitter.client = self.client
if self.in_lambda:
self.segment = xray_recorder.begin_subsegment(self.service_name)
else:
self.segment = xray_recorder.begin_segment(
self.service_name, sampling=True)
p = self.ctx.policy
xray_recorder.put_annotation('policy', p.name)
xray_recorder.put_annotation('resource', p.resource_type)
if self.ctx.options.account_id:
xray_recorder.put_annotation('account', self.ctx.options.account_id)
def random_sleep(max_seconds=10):
xray_recorder.put_annotation("RANDMOMLY_SLEEP_AT_WORK", "True")
sleep((random.randint(0, max_seconds) / 10))
def index():
request = app.current_request
json_log("Got a request", attrs=request.json_body)
if request.method == 'POST':
xray_recorder.put_annotation("url", "/")
xray_recorder.put_annotation("method", "POST")
messageRequest = request.json_body
xray_recorder.put_annotation('request_body', repr(request.json_body))
json_log("Message request {}\n".format(repr(request.json_body)),
attrs=request.json_body)
response = {'Message': '', 'Signature': []}
message = messageRequest['Message']
json_log("Message sent to swapcase service", attrs=request.json_body)
# try:
# subsegment = xray_recorder.begin_subsegment('external_request')
# req = requests.post(SWAPCASESERVICE, json={'Message':messageRequest['Message']}, timeout=2)
message = messageRequest['Message'].swapcase()
# xray_recorder.put_annotation('external_service_body', req.text)
# xray_recorder.put_annotation('call_external_service', req.status_code)
xray_recorder.end_subsegment()
#The external service may require anothey key soon, as below.
#req = requests.post(SWAPCASE_SERVICE, json={'Message':messageRequest['Message'],'ApiVersion':'2'}).json()
# except requests.exceptions.Timeout:
# #request took over 2 seconds, lets try again and hope not to go over 3 second lambda duration
# #SWAPCASESERVICE ApiVersion 1 sometimes takes up to 8 seconds
# json_log("External service took over 2 seconds. Retrying. By default swapcaser lambda has max 3 second runtime.",status='warning',attrs=request.json_body)
# req = requests.post(SWAPCASESERVICE, json={'Message':messageRequest['Message']})
# if req.status_code != 200:
# json_log('External service 500: Inspect "external_request" trace in AWS Xray','error', request.json_body)
# req.raise_for_status()
# req = req.json()
json_log("Swapcase service request complete", attrs=request.json_body)
response['Message'] = TEAMID + message
response['Signature'] = sign(
HASH,
response['Message']) #Sign message with my team hash (or HASH)
json_log("Response from sign function {}\n".format(repr(response)),
attrs=response)
return response
elif request.method == 'GET':
xray_recorder.put_annotation("url", "/")
xray_recorder.put_annotation("method", "GET")
# json_log('External service 500: Inspect "external_request" trace in AWS Xray','error', request.json_body)
# req.raise_for_status()
# req = req.json()
json_log("Swapcase service request complete", attrs=request.json_body)
response['Message'] = TEAMID + message
response['Signature'] = sign(
HASH,
response['Message']) #Sign message with my team hash (or HASH)
json_log("Response from sign function {}\n".format(repr(response)),
attrs=response)
return response
elif request.method == 'GET':
xray_recorder.put_annotation("url", "/")
xray_recorder.put_annotation("method", "GET")
return {'Status': 'OK'}
def get_recipients(alarm_name) -> List[str]:
xray_recorder.put_annotation('Alarm Name', alarm_name)
return [r.strip() for r in config.get(section="Recipients", option=alarm_name).split(',')]
def update_xray_annotations(
event_confidence,
event_criticality,
event_title,
remediation_type,
minimum_threat_score_for_remediation,
minimum_certainty_score_for_remediation,
event_source,
):
xray_recorder.put_annotation("event_confidence", event_confidence)
xray_recorder.put_annotation("event_criticality", event_criticality)
xray_recorder.put_annotation("event_title", event_title)
xray_recorder.put_annotation("remediation_type", remediation_type)
xray_recorder.put_annotation("minimum_threat_score_for_remediation",
minimum_threat_score_for_remediation)
xray_recorder.put_annotation(
"minimum_certainty_score_for_remediation",
minimum_certainty_score_for_remediation,
)
xray_recorder.put_annotation("event_source", event_source)
minimum_certainty_score_for_remediation = 99
# Next is the remediation type.
if remediation_type is None:
logger.critical(
"CRITICAL ERROR! unable to obtain remediation_type from the execution envrionment. Returned value is None."
)
raise RuntimeError(
"unable to obtain remediation_type from the execution envrionment.")
# I do not want to set a default, although a default of stop might make sense.
# Now let's add some annotations to the xray trace for this subsesgment.
if DEPLOYMENT_STAGE in ["dev", "test"]:
xray_recorder.put_annotation("minimum_threat_score_for_remediation",
minimum_threat_score_for_remediation)
xray_recorder.put_annotation(
"minimum_certainty_score_for_remediation",
minimum_certainty_score_for_remediation,
)
xray_recorder.put_annotation("remediation_type", remediation_type)
xray_recorder.end_subsegment()
# Looking good, lets get some information from the event and context objects
def update_xray_annotations(
event_confidence,
event_criticality,
event_title,
remediation_type,
url = os.getenv('IMAGE_URL',
'https://www.python.org/static/img/[email protected]')
bucket = os.getenv('DST_BUCKET', '1233343-src')
key = 'upload.png'
LOGGER = Logger()
if __name__ == '__main__':
# Start a segment if no segment exist
LOGGER.info('starting segment.')
segment = xray_recorder.begin_segment('BatchJob')
# This will add the key value pair to segment as it is active
LOGGER.info('adding annotation to segment.')
xray_recorder.put_annotation('XRayOnBatch', 'What sourcery is this?')
# This will add the key value pair to subsegment as it is activeimport platform
xray_recorder.put_metadata('Python Run-Time', platform.python_version())
if xray_recorder.is_sampled():
LOGGER.info('segment was sampled.')
xray_recorder.put_annotation('job_id', os.getenv('AWS_BATCH_JOB_ID'))
object_key = f"{id_generator()}.png"
LOGGER.info(f"downloading picture from {url}")
downloaded_file = requests.get(url)
LOGGER.info(f"Uploading {object_key} to bucket: {bucket}")
s3.put_object(Body=downloaded_file.content, Bucket=bucket, Key=object_key)
