def test_ALL(): #ALL tages
set_all = [{
'type': 'PERSON',
'topic': 'Donald Trump'
}, {
'type': 'GPE',
'topic': 'USA'
}, {
'type': 'ORG',
'topic': 'BBC'
}, {
'type': 'PERCENT',
'topic': '90%'
}, {
'type': 'LANGUAGE',
'topic': 'English'
}, {
'type': 'DATE',
'topic': 'Today'
}, {
'type': 'TIME',
'topic': '11:00'
}, {
'type': 'LOC',
'topic': 'Rocky Mountains'
}, {
'type': 'NORP',
'topic': 'Spanish'
}, {
'type': 'EVENT',
'topic': 'World War II'
}, {
'type': 'WORK_OF_ART',
'topic': 'Bible'
}, {
'type': 'MONEY',
'topic': '10,000,000'
}, {
'type': 'QUANTITY',
'topic': '20 kilometers'
}, {
'type': 'ORDINAL',
'topic': 'third'
}, {
'type': 'CARDINAL',
'topic': 'a thousand'
}]
assert spacyMat.spacyMatcher(text, '') == set_all
def test_Cardinal(
): #Numerals that do not fall under another type (not ordinal, quantity ..)
set_card = [{'type': 'CARDINAL', 'topic': 'a thousand'}]
assert spacyMat.spacyMatcher(text, 'CARDINAL') == set_card
def test_Ordinal(): #“first”, “second”, etc.
set_ord = [{'type': 'ORDINAL', 'topic': 'third'}]
assert spacyMat.spacyMatcher(text, 'ORDINAL') == set_ord
def test_Quantity(): #Measurements, as of weight or distance.
set_qty = [{'type': 'QUANTITY', 'topic': '20 kilometers'}]
assert spacyMat.spacyMatcher(text, 'QUANTITY') == set_qty
def test_Money(): #Monetary values, including unit.
set_money = [{'type': 'MONEY', 'topic': '10,000,000'}]
assert spacyMat.spacyMatcher(text, 'MONEY') == set_money
def test_WorkArt(): #Titles of books, songs, etc.
set_art = [{'type': 'WORK_OF_ART', 'topic': 'Bible'}]
assert spacyMat.spacyMatcher(text, 'WORK_OF_ART') == set_art
def test_Event(): #Named hurricanes, battles, wars, sports events, etc.
set_event = [{'type': 'EVENT', 'topic': 'World War II'}]
assert spacyMat.spacyMatcher(text, 'EVENT') == set_event
def test_GPE(): #GPE - Countries, cities, states.
set_gpe = [{'type': 'GPE', 'topic': 'USA'}]
assert spacyMat.spacyMatcher(text, 'GPE') == set_gpe
def test_Loc(): #Non-GPE locations, mountain ranges, bodies of water.
set_loc = [{'type': 'LOC', 'topic': 'Rocky Mountains'}]
assert spacyMat.spacyMatcher(text, 'LOC') == set_loc
def test_Time(): #Times smaller than a day.
set_time = [{'type': 'TIME', 'topic': '11:00'}]
assert spacyMat.spacyMatcher(text, 'TIME') == set_time
def test_Date(): #Absolute or relative dates or periods.
set_date = [{'type': 'DATE', 'topic': 'Today'}]
assert spacyMat.spacyMatcher(text, 'DATE') == set_date
def test_Language(): #Any named language.
set_lang = [{'type': 'LANGUAGE', 'topic': 'English'}]
assert spacyMat.spacyMatcher(text, 'LANGUAGE') == set_lang
def test_Percent(): #Percentage, including ”%“.
set_pct = [{'type': 'PERCENT', 'topic': '90%'}]
assert spacyMat.spacyMatcher(text, 'PERCENT') == set_pct
def test_Org(): #ORG - Companies, agencies, institutions, etc.
set_org = [{'type': 'ORG', 'topic': 'BBC'}]
assert spacyMat.spacyMatcher(text, 'ORG') == set_org
def test_Norp(): #Nationalities or religious or political groups.
set_norp = [{'type': 'NORP', 'topic': 'Spanish'}]
assert spacyMat.spacyMatcher(text, 'NORP') == set_norp
def lambda_handler(event, context):
## event must be a dict with a url key, and context can be nothing:
## lambda_handler({"url":"http://bbc.co.uk"}, "")
subsegment = xray_recorder.begin_subsegment('lambda_function: check URL')
logger.info(f'LambdaFunction: Checking we have a URL...')
try:
url = event['url']
except KeyError:
logger.info(f'LambdaFunction: URL is not present.')
return {"error": "No URL provided"}
xray_recorder.end_subsegment()
#### Adding "https://" to the URL if not present
logger.info(f'LambdaFunction: Checking URL has protocol...')
subsegment = xray_recorder.begin_subsegment(
'lambda_function: check URL protocol')
if (not url.startswith('https://') and not url.startswith('http://')):
url = 'https://' + url
xray_recorder.end_subsegment()
logger.info(f'LambdaFunction: Validating URL...')
subsegment = xray_recorder.begin_subsegment(
'lambda_function: validate URL')
if (not validators.url(url)):
logger.info(f'LambdaFunction: URL is not valid.')
return {"error": "The url was bad"}
xray_recorder.end_subsegment()
#### Define the object skeleton
object = {"url": url, "results": []}
# CredibilityScore
logger.info(f'LambdaFunction: Trying to get credibility score...')
credibilityresult = {}
try:
credibilityresult = cr.getCredibilityScore(url)
object['results'].append(credibilityresult)
except Exception as e:
logger.info(f'LambdaFunction: Could not get Credibility Score.')
logger.info(e)
credibilityresult = {
'type': 'credibility',
'outcome': {
"error": "The credibility score was not available."
}
}
object['results'].append(credibilityresult)
# SentimentAnalisys
logger.info(f'LambdaFunction: Trying to get sentimentAnalysis score...')
sentanalysisresult = {}
try:
sentanalysisresult = sa.sentimentAnalysis(url)
if sentanalysisresult['text'] == '-1':
logger.info(
f'LambdaFunction: sentimentAnalysis returned -1, dumping:')
logger.info(sentanalysisresult)
object['article'] = {
'error': "The article summary could not be generated"
}
object['results'].append({
'type': 'polarity',
"outcome": {
"error": "The polarity score could not be calculated."
}
})
object['results'].append({
'type': 'objectivity',
"outcome": {
"error": "The objectivity score could not be calculated."
}
})
else:
object['article'] = {
'header': sentanalysisresult['header'],
'summary': sentanalysisresult['summary'],
'keywords': sentanalysisresult['keywords'],
'image': sentanalysisresult['image']
}
object['results'].append({
'type': 'polarity',
'outcome': {
"score": sentanalysisresult['polarity']
}
})
object['results'].append({
'type': 'objectivity',
'outcome': {
"score": abs(1 - sentanalysisresult['subjectivity'])
}
})
except Exception as e:
logger.info(f'LambdaFunction: Could not get sentimentAnalysis Score.')
exception_type, exception_value, exception_traceback = sys.exc_info()
traceback_string = traceback.format_exception(exception_type,
exception_value,
exception_traceback)
err_msg = json.dumps({
"errorType": exception_type.__name__,
"errorMessage": str(exception_value),
"stackTrace": traceback_string
})
logger.error(err_msg)
sentanalysisresult = {'text': -1}
object['article'] = {
'error': "The article summary could not be generated"
}
object['results'].append({
'type': 'polarity',
"outcome": {
"error": "The polarity score could not be calculated."
}
})
object['results'].append({
'type': 'objectivity',
"outcome": {
"error": "The objectivity score could not be calculated."
}
})
# BiasScore
logger.info(f'LambdaFunction: Trying to get bias score...')
if sentanalysisresult[
'text'] != '-1': #### if there is no POL or SUBJ, getBiasScore will not be called
if 'error' in credibilityresult['outcome']:
cred_input = -1
else:
cred_input = credibilityresult['outcome']['score']
try:
biasscoreresult = bs.getBiasScore(
cred_input, sentanalysisresult['polarity'],
sentanalysisresult['subjectivity'])
object['results'].append(biasscoreresult)
except Exception as e:
logger.info(f'LambdaFunction: Could not get Bias Score.')
logger.info(e)
object['results'].append({
'type': 'bias',
'outcome': {
"error": "The bias score was not available."
}
})
else:
object['results'].append({
'type': 'bias',
'outcome': {
"error": "The bias score was not available."
}
})
# SpacyMatcher
logger.info(f'LambdaFunction: Trying to get Spacy Matcher...')
if sentanalysisresult['text'] != '-1':
try:
list_objs = sm.spacyMatcher(sentanalysisresult['text'],
'') ## ALL Tags
object['article']['topics'] = list_objs
except Exception as e:
logger.info(f'LambdaFunction: Could not get Topics.')
logger.info(e)
object['article']['topics'] = {"error": "No topics available."}
#### JSON to return:
# {
# 'url':'https://www.theguardian.com/world/2020/',
# 'article' : {
# 'header' : 'PM admits failings as England's Covid contact',
# 'summary' : Boris Johnson and his chief scientific ...',
# 'keywords' : ['Boris Johnson', 'Brexit'],
# 'topics': [{'type': 'DATE', 'topic': 'Today'}]}}
# },
# 'results' : [
# { 'type' : 'credibility', 'outcome': {'score': 100.0, 'source ...
# { 'type' : 'polarity', 'outcome': {'score': 0.108126295001 ...
# { 'type' : 'objectivity', 'outcome': {'score': 0.487846736596 ...
# { 'type' : 'bias', 'outcome': {'score': 20.67598528015 ...
# ]
# }
return object
def test_Person(): # PERSON - People, including fictional.
set_person = [{'type': 'PERSON', 'topic': 'Donald Trump'}]
assert spacyMat.spacyMatcher(text, 'PERSON') == set_person