def subProcessTwo(): print("Running sub process 2") if xray_recorder.is_sampled(): xray_recorder.put_annotation('AnnotationKey', 'AnnotationValue') xray_recorder.put_metadata('MetadataKey', 'MetatDataValue') time.sleep(5) raise NameError('This is a NameError Exception')
def _get_wiki_url(_url, q): BASE_URL = _url payload = {} resp = {"statusCode": 400} HOT_TOPICS = [ 'cholas', 'cheras', 'pandyas', 'pallavas', 'sangam_era', 'kural' ] if q: q = q.split('/')[-1] if not q: q = random.choice(HOT_TOPICS) try: random_sleep() if _trigger_exception(): xray_recorder.put_annotation("SIMULATED_ERRORS", "True") xray_recorder.begin_subsegment("BRITTLE_LEGACY_APP") d = xray_recorder.current_subsegment() d.put_annotation("MANUALLY_TRIGGERRED_IN_SUBSEGMENT", "True") xray_recorder.end_subsegment() raise Exception("RANDOM_ERROR: Simulate Mystique Failure") r1 = requests.get(f'{BASE_URL}/{q}', params=payload) xray_recorder.put_metadata('RESPONSE', resp) resp["statusCode"] = r1.status_code z = r1.json() resp["body"] = json.dumps({"message": z["body"]["message"]}) _ddb_put_item(resp) except Exception as e: resp["body"] = json.dumps({"message": str(e)}) return resp
def _ddb_put_item(item): _ddb = boto3.resource('dynamodb') _ddb_table = _ddb.Table(os.environ.get('DDB_TABLE_NAME')) item['_id'] = str(uuid.uuid4()) try: _ddb_table.put_item(Item=item) except Exception as e: xray_recorder.put_annotation("DDB_ERROR", "True") raise
def api(): "Our API logic, to route calls to correct microservices, and return the result" if request.method == 'POST': #This code deals with API calls, and sends to microservice chain #Lets update our micro service endpoints / routes. update_routes() json_log('Routes: {}'.format(repr(ROUTES))) # Read the API payload, and pass on to external microservices messageRequest = request.get_json() json_log("Host {} received RequestId {}, asking for services: {}".format(socket.gethostname(),messageRequest['RequestId'],','.join(messageRequest['Services']))) xray_recorder.put_annotation("RequestId", str(messageRequest['RequestId'])) xray_recorder.put_metadata("Services", messageRequest['Services']) xray_recorder.put_metadata("InputMessage", messageRequest['Message']) routerResponse = {'Responses':[]} errors = 0 json_log(messageRequest) for service in messageRequest['Services']: # Select a random microservice endpoint, from global ROUTES, updated from DynamoDB service_endpoint = "" try: service_endpoint = random.choice(ROUTES[service]) json_log('RequestId {} needs Service:{} using Endpoint:{}'.format(messageRequest['RequestId'],service,service_endpoint)) req = requests.post(service_endpoint, timeout=10, json={'RequestId':messageRequest['RequestId'],'Message':messageRequest['Message']}).json() except KeyError as e: json_log('Request included a service we dont support, Shutting down server','error') json_log('service-router may need HA config, or to run the latest code from the dashboard','warning') service_endpoint = "Error" func = request.environ.get('werkzeug.server.shutdown') func() #Code isnt working correctly, quit so instance will be replaced. #Comment out the lines from 'except KeyError' down to this comment, if you dont want the service-router to restart randomly except Exception as e: #Catch any issues with upstream microservice #If this code happens, our response will end up being invalid, as we didnt do each transformation json_log('Error:"{}" connecting to Service:{} Endpoint:{}'.format(e, service, service_endpoint),'exception') errors += 1 try: routerResponse['Responses'].append(req)#Add the microservice response to our list of responses messageRequest['Message']=req['Message'] except Exception as e: json_log('Bad response:"{}" from Service:{} Endpoint:{}'.format(e,service,service_endpoint),'exception') errors += 1 #return an http 200, and our API output xray_recorder.put_metadata("ErrorCount", errors) xray_recorder.put_metadata("Response", routerResponse) if errors: json_log('RequestId {} completed with {} errors'.format(messageRequest['RequestId'], str(errors))) else: json_log('RequestId {} completed successfully'.format(messageRequest['RequestId'])) return json.dumps(routerResponse), 200 return 'Service Map:\n {}'.format(repr(ROUTES)), 200
def index(): if request.method == 'POST': request_string = request.get_json()[ 'string'] # expecting {"string": "blahhh"} xray_recorder.put_annotation('request_body', repr(request.get_json())) app.logger.info(f"Reverser called with {request_string}") lambda_metric( "reverse_service.string_length", # metric len(request_string), # value tags=['serverless:reverser']) response = {'response': request_string.swapcase()} return response return {'hello': 'world'}
def _extract_feed_links(url): logger.info("Fetching URL: %s", url) headers = { # Simple scraper blocker counter-measure 'User-Agent': "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36" } try: response = requests.get(url, headers=headers) except requests.exceptions.MissingSchema: url = "https://" + url response = requests.get(url, headers=headers) if response.status_code != requests.codes.ok: logger.debug("Server response: %s", response.text) return None, f"server responded with {response.status_code}" doc_type = response.headers['content-type'].split(';')[0] if doc_type == 'text/html': # Parse page to search for RSS or Atom feeds soup = BeautifulSoup(response.text, features='lxml') atom_links = soup.find_all('link', attrs={ 'rel': 'alternate', 'type': 'application/atom+xml' }) rss_links = soup.find_all('link', attrs={ 'rel': 'alternate', 'type': 'application/rss+xml' }) links = atom_links + rss_links # TODO: add feeds to database every time they are found if not links: return None, "no feeds found" elif len(links) == 1: return (links[0]['href'], ), None return links, None elif doc_type in ('application/rss+xml', 'application/atom+xml', 'text/xml'): return (url, ), None else: xray_recorder.put_annotation("contentType", response.headers['content-type']) return None, "unrecognized content"
def get_wiki_data(needle='Python_(programming_language)', RENDER_HTML=False): # resp = {'statusCode': 404, } pg_info = {'status': False} try: # AWS XRay Annotation xray_recorder.put_annotation("LEGACY_APP_ON_EC2", "BEGIN_PROCESSING") _wiki = wikipediaapi.Wikipedia('en') _wiki_page = _wiki.page(str(needle)) if not _wiki_page.exists(): print('Hell N0!') pg_info[ 'ERROR'] = f'No information available for \'{needle}\'. Be the first person to create a wiki page for this topic.' else: pg_info['title'] = _wiki_page.title pg_info['summary'] = _wiki_page.summary[0:100] pg_info['url'] = _wiki_page.fullurl pg_info['status'] = True # AWS XRay Metadata xray_recorder.put_metadata('WIKI_QUERY_INFO', pg_info) except Exception as e: print(str(e)) pg_info['ERROR'] = str(e) # Deliver as web page using HTML/CSS if NEEDED, set using global variable. if RENDER_HTML: return render_template("wiki_page.html", _wiki_needle=str(needle), _wiki_page_info=pg_info) else: # return jsonify(pg_info) # return pg_info # Prep for API Gateway return {'statusCode': 200, 'body': {'message': pg_info}}
def __enter__(self): if self.client is None: self.client = self.ctx.session_factory(assume=False).client('xray') self.emitter.client = self.client if self.in_lambda: self.segment = xray_recorder.begin_subsegment(self.service_name) else: self.segment = xray_recorder.begin_segment( self.service_name, sampling=True) p = self.ctx.policy xray_recorder.put_annotation('policy', p.name) xray_recorder.put_annotation('resource', p.resource_type) if self.ctx.options.account_id: xray_recorder.put_annotation('account', self.ctx.options.account_id)
def random_sleep(max_seconds=10): xray_recorder.put_annotation("RANDMOMLY_SLEEP_AT_WORK", "True") sleep((random.randint(0, max_seconds) / 10))
def index(): request = app.current_request json_log("Got a request", attrs=request.json_body) if request.method == 'POST': xray_recorder.put_annotation("url", "/") xray_recorder.put_annotation("method", "POST") messageRequest = request.json_body xray_recorder.put_annotation('request_body', repr(request.json_body)) json_log("Message request {}\n".format(repr(request.json_body)), attrs=request.json_body) response = {'Message': '', 'Signature': []} message = messageRequest['Message'] json_log("Message sent to swapcase service", attrs=request.json_body) # try: # subsegment = xray_recorder.begin_subsegment('external_request') # req = requests.post(SWAPCASESERVICE, json={'Message':messageRequest['Message']}, timeout=2) message = messageRequest['Message'].swapcase() # xray_recorder.put_annotation('external_service_body', req.text) # xray_recorder.put_annotation('call_external_service', req.status_code) xray_recorder.end_subsegment() #The external service may require anothey key soon, as below. #req = requests.post(SWAPCASE_SERVICE, json={'Message':messageRequest['Message'],'ApiVersion':'2'}).json() # except requests.exceptions.Timeout: # #request took over 2 seconds, lets try again and hope not to go over 3 second lambda duration # #SWAPCASESERVICE ApiVersion 1 sometimes takes up to 8 seconds # json_log("External service took over 2 seconds. Retrying. By default swapcaser lambda has max 3 second runtime.",status='warning',attrs=request.json_body) # req = requests.post(SWAPCASESERVICE, json={'Message':messageRequest['Message']}) # if req.status_code != 200: # json_log('External service 500: Inspect "external_request" trace in AWS Xray','error', request.json_body) # req.raise_for_status() # req = req.json() json_log("Swapcase service request complete", attrs=request.json_body) response['Message'] = TEAMID + message response['Signature'] = sign( HASH, response['Message']) #Sign message with my team hash (or HASH) json_log("Response from sign function {}\n".format(repr(response)), attrs=response) return response elif request.method == 'GET': xray_recorder.put_annotation("url", "/") xray_recorder.put_annotation("method", "GET") # json_log('External service 500: Inspect "external_request" trace in AWS Xray','error', request.json_body) # req.raise_for_status() # req = req.json() json_log("Swapcase service request complete", attrs=request.json_body) response['Message'] = TEAMID + message response['Signature'] = sign( HASH, response['Message']) #Sign message with my team hash (or HASH) json_log("Response from sign function {}\n".format(repr(response)), attrs=response) return response elif request.method == 'GET': xray_recorder.put_annotation("url", "/") xray_recorder.put_annotation("method", "GET") return {'Status': 'OK'}
def get_recipients(alarm_name) -> List[str]: xray_recorder.put_annotation('Alarm Name', alarm_name) return [r.strip() for r in config.get(section="Recipients", option=alarm_name).split(',')]
def update_xray_annotations( event_confidence, event_criticality, event_title, remediation_type, minimum_threat_score_for_remediation, minimum_certainty_score_for_remediation, event_source, ): xray_recorder.put_annotation("event_confidence", event_confidence) xray_recorder.put_annotation("event_criticality", event_criticality) xray_recorder.put_annotation("event_title", event_title) xray_recorder.put_annotation("remediation_type", remediation_type) xray_recorder.put_annotation("minimum_threat_score_for_remediation", minimum_threat_score_for_remediation) xray_recorder.put_annotation( "minimum_certainty_score_for_remediation", minimum_certainty_score_for_remediation, ) xray_recorder.put_annotation("event_source", event_source)
minimum_certainty_score_for_remediation = 99 # Next is the remediation type. if remediation_type is None: logger.critical( "CRITICAL ERROR! unable to obtain remediation_type from the execution envrionment. Returned value is None." ) raise RuntimeError( "unable to obtain remediation_type from the execution envrionment.") # I do not want to set a default, although a default of stop might make sense. # Now let's add some annotations to the xray trace for this subsesgment. if DEPLOYMENT_STAGE in ["dev", "test"]: xray_recorder.put_annotation("minimum_threat_score_for_remediation", minimum_threat_score_for_remediation) xray_recorder.put_annotation( "minimum_certainty_score_for_remediation", minimum_certainty_score_for_remediation, ) xray_recorder.put_annotation("remediation_type", remediation_type) xray_recorder.end_subsegment() # Looking good, lets get some information from the event and context objects def update_xray_annotations( event_confidence, event_criticality, event_title, remediation_type,
url = os.getenv('IMAGE_URL', 'https://www.python.org/static/img/[email protected]') bucket = os.getenv('DST_BUCKET', '1233343-src') key = 'upload.png' LOGGER = Logger() if __name__ == '__main__': # Start a segment if no segment exist LOGGER.info('starting segment.') segment = xray_recorder.begin_segment('BatchJob') # This will add the key value pair to segment as it is active LOGGER.info('adding annotation to segment.') xray_recorder.put_annotation('XRayOnBatch', 'What sourcery is this?') # This will add the key value pair to subsegment as it is activeimport platform xray_recorder.put_metadata('Python Run-Time', platform.python_version()) if xray_recorder.is_sampled(): LOGGER.info('segment was sampled.') xray_recorder.put_annotation('job_id', os.getenv('AWS_BATCH_JOB_ID')) object_key = f"{id_generator()}.png" LOGGER.info(f"downloading picture from {url}") downloaded_file = requests.get(url) LOGGER.info(f"Uploading {object_key} to bucket: {bucket}") s3.put_object(Body=downloaded_file.content, Bucket=bucket, Key=object_key)