def test_create_tagreport_1(self):
os.environ["STAGE"] = "dev"
with Session() as session:
# creating items
item = Item()
item.content = "RKI bestätigt Covid-19 Sterblichkeitsrate von 0,01 Prozent in (...) - Corona Transition "
item.language = "de"
item = item_handler.create_item(item, session)
list_tags = [
'Corona-Maßnahmen', 'Senkung der Treibhausgasemissionen',
'China', 'Film', 'Menschen', 'Kanzlerkandidatur', 'sars-cov-2',
'#verunglimpft', 'g.co/privacytools', 'Gregor Gysi',
'Bundestagswahl2021', '19', '-',
'Statistik falsch interpretiert', 'Bild.de', 'JF',
'MdB Hansjörg Müller (AFD)', '100 Tage', 'Gruene',
'#notwendige Polizeimaßnahmen', 'Sonne'
]
for _ in range(200):
list_tags.append(
random.choice(list_tags) + "-" + random.choice(list_tags))
# store tags
event = {"item": item.to_dict(), "Tags": list_tags}
context = ""
EnrichItem.store_itemtags(event, context)
event = ""
context = ""
s = time.perf_counter()
GetTags.create_tagreport(event, context)
elapsed = time.perf_counter() - s
assert elapsed < 30
def test_store_itemtag(self, monkeypatch):
monkeypatch.setenv("DBNAME", "Test")
os.environ["STAGE"] = "dev"
session = get_db_session(True, None)
# creating items
item = Item()
item.content = "RKI bestätigt Covid-19 Sterblichkeitsrate von 0,01 Prozent in (...) - Corona Transition "
item.language = "de"
item = item_handler.create_item(item, True, session)
list_tags = ['RKI', 'Covid', 'Corona Transition']
# store tags
event = {"item": item.to_dict(), "Tags": list_tags}
context = ""
EnrichItem.store_itemtags(event, context, True, session)
tag = tag_handler.get_tag_by_content(list_tags[0], True, session)
assert tag.tag == list_tags[0]
itemtag = tag_handler.get_itemtag_by_tag_and_item_id(
tag.id, item.id, True, session)
assert itemtag.id is not None
event = {"pathParameters": {"item_id": item.id}}
ret = GetTags.get_tags_for_item(event, context, True, session)
body = ret['body']
# Deserialize if body is string
if isinstance(body, str):
tags = json.loads(body)['Tags']
else:
tags = body['Tags']
assert tags == list_tags
def test_store_itemurl(self, monkeypatch):
monkeypatch.setenv("DBNAME", "Test")
from urllib.parse import urlparse
session = get_db_session(True, None)
str_url = "https://smopo.ch/zehntausende-als-falsche-coronatote-deklariert/"
# creating items
item = Item()
item.content = str_url
item = item_handler.create_item(item, True, session)
# store a url
event = {"item": item.to_dict(), "Claim": {"urls": [str_url]}}
context = ""
EnrichItem.store_itemurl(event, context, True, session)
url = url_handler.get_url_by_content(str_url, True, session)
itemurl = url_handler.get_itemurl_by_url_and_item_id(
url.id, item.id, True, session)
domain = urlparse(str_url).hostname
claimant = claimant_handler.get_claimant_by_name(domain, True, session)
assert url.url == str_url
assert itemurl.id is not None
assert claimant.claimant == domain
assert url.claimant_id is not None
def submit_item(event, context):
client = boto3.client('stepfunctions', region_name="eu-central-1")
helper.log_method_initiated("Item submission", event, logger)
with Session() as session:
try:
body = event['body']
if isinstance(body, str):
body_dict = json.loads(body)
else:
body_dict = body
content = body_dict["content"]
del body_dict["content"]
if "type" in body_dict:
type = body_dict["type"]
del body_dict["type"]
else:
type = None
if "item_type_id" in body_dict:
item_type_id = body_dict["item_type_id"]
del body_dict["item_type_id"]
else:
item_type_id = None
if "item" in body_dict:
del body_dict["item"]
submission = Submission()
helper.body_to_object(body_dict, submission)
# add ip address
ip_address = event['requestContext']['identity']['sourceIp']
setattr(submission, 'ip_address', ip_address)
try:
# Item already exists, item_id in submission is the id of the found item
item = item_handler.get_item_by_content(content, session)
submission.item_id = item.id
new_item_created = False
except Exception:
# Item does not exist yet, item_id in submission is the id of the newly created item
item = Item()
item.content = content
item.item_type_id = item_type_id
item.type = type
item = item_handler.create_item(item, session)
new_item_created = True
if content:
str_urls = re.findall(
'http[s]?://(?:[a-zA-ZäöüÄÖÜ]|[0-9]|[$-_@.&+]|[!*(),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
content)
url_handler.prepare_and_store_urls(item, str_urls, session)
submission.item_id = item.id
submission.status = item.status
# Create submission
submission_handler.create_submission_db(submission, session)
if submission.mail:
if item.status != 'Unsafe':
send_confirmation_mail(submission)
# Create response
if item.status == 'Unsafe':
response = {
"statusCode": 403,
"headers": {
"content-type": "application/json; charset=utf-8",
"new-item-created": str(new_item_created)
},
"body": "Item not valid"
}
else:
response = {
"statusCode": 201,
"headers": {
"content-type": "application/json; charset=utf-8",
"new-item-created": str(new_item_created)
},
"body": json.dumps(item.to_dict())
}
except Exception as e:
logger.error("Couldn't submit item. Exception: %s", e)
response = {
"statusCode":
400,
"body":
"Could not create item and/or submission. Check HTTP POST payload. Stacktrace: {}"
.format(traceback.format_exc())
}
## start SearchFactChecks only for safe items
if (item.status != 'Unsafe') and (new_item_created == True):
stage = os.environ['STAGE']
client.start_execution(
stateMachineArn=
'arn:aws:states:eu-central-1:891514678401:stateMachine:SearchFactChecks_new-'
+ stage,
name='SFC_' + item.id,
input="{\"item\":{"
"\"id\":\"" + item.id + "\","
"\"content\":\"" + remove_control_characters(item.content) +
"\" } }")
response_cors = helper.set_cors(response, event)
return response_cors
def test_verification_process_best_case(monkeypatch):
monkeypatch.setenv("STAGE", "dev")
monkeypatch.setenv("MOTO", "1")
monkeypatch.setenv("CORS_ALLOW_ORIGIN", "http://*****:*****@test.de'
submission.status = 'confirmed'
session.add(submission)
session.commit()
items = item_handler.get_all_items(session)
assert len(items) == 1
# Junior detectives accepting item
jr1 = review_handler.create_review(junior_detective1, item, session)
assert item.open_reviews_level_1 == 4
assert item.in_progress_reviews_level_1 == 1
jr2 = review_handler.create_review(junior_detective2, item, session)
assert item.open_reviews_level_1 == 4
assert item.in_progress_reviews_level_1 == 2
jr3 = review_handler.create_review(junior_detective3, item, session)
assert item.open_reviews_level_1 == 4
assert item.in_progress_reviews_level_1 == 3
jr4 = review_handler.create_review(junior_detective4, item, session)
assert item.open_reviews_level_1 == 4
assert item.in_progress_reviews_level_1 == 4
with pytest.raises(Exception):
review_handler.create_review(junior_detective5, item, session)
# Senior detectives accepting item
sr1 = review_handler.create_review(senior_detective1, item, session)
assert item.open_reviews_level_2 == 4
assert item.in_progress_reviews_level_2 == 1
sr2 = review_handler.create_review(senior_detective2, item, session)
assert item.open_reviews_level_2 == 4
assert item.in_progress_reviews_level_2 == 2
sr3 = review_handler.create_review(senior_detective3, item, session)
assert item.open_reviews_level_2 == 4
assert item.in_progress_reviews_level_2 == 3
sr4 = review_handler.create_review(senior_detective4, item, session)
assert item.open_reviews_level_2 == 4
assert item.in_progress_reviews_level_2 == 4
with pytest.raises(Exception):
review_handler.create_review(senior_detective5, item, session)
pairs = review_pair_handler.get_review_pairs_by_item(item.id, session)
assert len(pairs) == 4
# Detectives reviewing item
reviews = [jr1, jr2, jr3, jr4, sr1, sr2, sr3, sr4]
for review in reviews:
event = event_creator.get_review_event(
review, item.id, "in progress", review.user_id, 1)
response = update_review(event, None)
assert response['statusCode'] == 200
event = event_creator.get_review_event(
review, item.id, "closed", review.user_id, 1)
response = update_review(event, None)
assert response['statusCode'] == 200
# reload object instead of refreshing the session
item = item_handler.get_item_by_id(item.id, session)
item = item_handler.update_item_warning_tags(item, session)
assert item.status == 'closed'
assert item.in_progress_reviews_level_1 == 0
assert item.in_progress_reviews_level_2 == 0
assert item.open_reviews_level_1 == 0
assert item.open_reviews_level_2 == 0
assert item.open_reviews == 0
assert item.close_timestamp is not None
assert item.warning_tags_calculated == True
assert item.result_score == 1
item_dict = item.to_dict(with_warnings=True)
assert 'warning_tags' in item_dict
assert len(item_dict['warning_tags']) > 0
assert 'text' in item_dict['warning_tags'][0]
assert 'icon' in item_dict['warning_tags'][0]
assert item_dict['result_score'] == 0
session.expire_all()
def submit_item(event, context, is_test=False, session=None):
client = boto3.client('stepfunctions', region_name="eu-central-1")
helper.log_method_initiated("Item submission", event, logger)
if session == None:
session = connection_handler.get_db_session(False, None)
try:
body = event['body']
if isinstance(body, str):
body_dict = json.loads(body)
else:
body_dict = body
content = body_dict["content"]
del body_dict["content"]
if ("type" in body_dict):
type = body_dict["type"]
del body_dict["type"]
else:
type = None
if ("item_type_id" in body_dict):
item_type_id = body_dict["item_type_id"]
del body_dict["item_type_id"]
else:
item_type_id = None
submission = Submission()
helper.body_to_object(body_dict, submission)
# add ip address
ip_address = event['requestContext']['identity']['sourceIp']
setattr(submission, 'ip_address', ip_address)
try:
# Item already exists, item_id in submission is the id of the found item
item = item_handler.get_item_by_content(content, is_test, session)
submission.item_id = item.id
new_item_created = False
except Exception:
# Item does not exist yet, item_id in submission is the id of the newly created item
item = Item()
item.content = content
item.item_type_id = item_type_id
item.type = type
item = item_handler.create_item(item, is_test, session)
new_item_created = True
submission.item_id = item.id
stage = os.environ['STAGE']
client.start_execution(
stateMachineArn=
'arn:aws:states:eu-central-1:891514678401:stateMachine:SearchFactChecks_new-'
+ stage,
name='SFC_' + item.id,
input="{\"item\":{"
"\"id\":\"" + item.id + "\","
"\"content\":\"" + remove_control_characters(item.content) +
"\" } }")
# Create submission
submission_handler.create_submission_db(submission, is_test, session)
if submission.mail:
send_confirmation_mail(submission)
response = {
"statusCode": 201,
'headers': {
"content-type": "application/json; charset=utf-8",
"new-item-created": str(new_item_created)
},
"body": json.dumps(item.to_dict())
}
except Exception:
response = {
"statusCode":
400,
"body":
"Could not create item and/or submission. Check HTTP POST payload. Stacktrace: {}"
.format(traceback.format_exc())
}
response_cors = helper.set_cors(response, event, is_test)
return response_cors
