def query_runner(event, context):
"""Triggered from a message on a Cloud Pub/Sub topic.
Args:
event (dict): Event payload.
context (google.cloud.functions.Context): Metadata for the event.
"""
pubsub_message = base64.b64decode(event['data']).decode('utf-8')
if pubsub_message == "export":
project_id = os.environ.get('GCP_PROJECT', '')
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(
project_id, os.environ.get('TABLES_LIST_TOPIC_NAME', ''))
database = os.environ.get('SQL_DB', '')
sql_user = get_secret(project_id, "sql_user")
sql_pass = get_secret(project_id, "sql_pass")
sql_connection_name = get_secret(project_id, "sql_connection_name")
db = sqlalchemy.create_engine(
sqlalchemy.engine.url.URL(
drivername="mysql+pymysql",
username=sql_user,
password=sql_pass,
database=database,
query={
"unix_socket": "/cloudsql/{}".format(sql_connection_name)},
),
pool_size=1,
# Temporarily exceeds the set pool_size if no connections are available.
max_overflow=1,
pool_timeout=30, # 30 seconds
pool_recycle=1800 # 30 minutes
)
stmt = sqlalchemy.text(os.environ.get('SQL_QUERY'))
try:
with db.connect() as conn:
result = conn.execute(stmt).fetchall()
export_tables = [r for r, in result]
# Publish list of tables into Pub/Sub.
data = ",".join(export_tables)
data = data.encode("utf-8")
# Set max number of function re-runs.
future = publisher.publish(
topic_path, data, batch_no="1", max_batches=str(os.environ.get('MAX_BATCHES', 5)))
print(future.result())
except Exception as e:
raise RuntimeError(
"Failed to execute statement. Exception: {}".format(e))
def test_list_secrets(capsys, secret):
project_id, secret_id, _ = secret
secret = get_secret(project_id, secret_id)
list_secrets(project_id)
out, _ = capsys.readouterr()
assert "Found secret: {}".format(secret.name) in out
def get_credibility_score(url):
"""
Input: URL
Output: Credibility Score, Category and Source
This function calls the API Gate Source Credibility passing the URL to
be analysed. It returns the Score, Category (Left Center, Fake News, ...)
and the Source. If the website is reated in the source 'Media Bias/Fact
Check', this source is then used. Otherwise it will a different source that
rated the website, or return "-1" in case no rating is found.
"""
p_score = 0
p_category = ''
p_source = ''
return_dict = {'type': 'credibility', 'outcome': {'score': -1}}
aws_secrets = secrets.get_secret("prod/getCredibilityScore/GATEKey",
"eu-west-2")
if "error" in aws_secrets:
return return_dict
gate_cred_api = "https://cloud-api.gate.ac.uk/process-document/source-credibility?annotations=:DomainCredibility"
cred_txt = requests.post(gate_cred_api,
data=url,
headers={'Content-Type': 'text/plain'},
auth=(aws_secrets['apiid'],
aws_secrets['apikey']))
rel_json = cred_txt.json()
if (rel_json['entities'] != {}):
for item in rel_json['entities']['DomainCredibility']:
p_score = item.get('credibility-score', 0)
p_category = item.get('credibility-category', '')
p_source = item.get('credibility-source', '')
return_dict = {
'type': 'credibility',
'outcome': {
'score': p_score,
'source': p_source,
'category': p_category
}
}
if item['credibility-source'] == 'Media Bias/Fact Check':
return return_dict
else:
return_dict = {
'type': 'credibility',
'outcome': {
"error": "The credibility score was not available."
}
}
return return_dict
return return_dict
def test_list_secrets_with_filter(capsys, secret):
project_id, secret_id, _ = secret
unlabeled = get_secret(project_id, secret_id)
list_secrets_with_filter(project_id, "labels.secretmanager:rocks")
out, _ = capsys.readouterr()
assert "Found secret: {}".format(unlabeled.name) not in out
labeled = update_secret(project_id, secret_id)
assert labeled.labels["secretmanager"] == "rocks"
list_secrets_with_filter(project_id, "labels.secretmanager:rocks")
out, _ = capsys.readouterr()
assert "Found secret: {}".format(labeled.name) in out
def get_connection(
self
): # function to get the connection string using: pymysql.connect(host, username, password, database)
if environ.get("ENVIRONMENT") == "prod":
host, username, password, db_name = get_secret()[0:5]
else:
host, username, password, db_name = environ.get(
"DB_HOST2"), environ.get("DB_USER2"), environ.get(
"DB_PW2"), environ.get("DB_NAME2")
try:
db_connection = pymysql.connect(host, username, password, db_name)
print("Got connection")
logger.info("Load connection successful LOL")
return db_connection
except Exception as error:
logger.critical("Load connection failed LOL")
print(f"didn't work lol {error}")
def test_get_secret(client, secret):
project_id, secret_id = secret
snippet_secret = get_secret(project_id, secret_id)
assert secret_id in snippet_secret.name
import boto3
from get_user import fetch_user
from delete_user import delete_user
from get_user_info import fetch_user_info
from create_encrypted_user import encrypt_item
from get_secret import get_secret
awsCmkId = get_secret()
table = boto3.resource("dynamodb").Table('userInfo')
USERNAME = str(input("please enter your username: "******"testing get_user on " + str(USERNAME) + " returns: " + str(fetched))
def test_get_user_info(USERNAME, awsCmkId, table):
# get_user_info test
DATA = str(input("please enter specific data needed: "))
info = fetch_user_info(USERNAME, DATA, awsCmkId, table)
print("getting " + str(DATA) + " from " + str(USERNAME) + " returns: " +
str(info))
def test_create_encrypted_user(USERNAME, awsCmkId, table):
PASSWORD = str(input("please enter your password: "******"please enter your first name: "))
MIDDLE_INITIAL = str(input("please enter your middle initial: "))
LAST_NAME = str(input("please enter your last name: "))
def test_get_secret():
key = "prod/test"
region = "eu-west-2"
result = secrets.get_secret(key, region)
assert result == {'abc': '123'}
def test_get_bad_secret():
key = "prod/badtest"
region = "eu-west-2"
result = secrets.get_secret(key, region)
assert result == {'error': 'Resource Not Found'}