def setUp(self):
super().setUp()
self.app = Flask(__name__)
FlaskInstrumentor().instrument_app(self.app)
self._common_initialization()
def setUp(self):
super().setUp()
Configuration._instance = None # pylint: disable=protected-access
Configuration.__slots__ = [] # pylint: disable=protected-access
FlaskInstrumentor().instrument()
self.app = flask.Flask(__name__)
self._common_initialization()
def test_uninstrument(self):
resp = self.client.get("/hello/123")
self.assertEqual(200, resp.status_code)
self.assertEqual([b"Hello: 123"], list(resp.response))
span_list = self.memory_exporter.get_finished_spans()
self.assertEqual(len(span_list), 1)
FlaskInstrumentor().uninstrument_app(self.app)
resp = self.client.get("/hello/123")
self.assertEqual(200, resp.status_code)
self.assertEqual([b"Hello: 123"], list(resp.response))
span_list = self.memory_exporter.get_finished_spans()
self.assertEqual(len(span_list), 1)
def test_uninstrument(self):
# pylint: disable=access-member-before-definition
resp = self.client.get("/hello/123")
self.assertEqual(200, resp.status_code)
self.assertEqual([b"Hello: 123"], list(resp.response))
span_list = self.memory_exporter.get_finished_spans()
self.assertEqual(len(span_list), 1)
FlaskInstrumentor().uninstrument()
self.app = flask.Flask(__name__)
self.app.route("/hello/<int:helloid>")(self._hello_endpoint)
# pylint: disable=attribute-defined-outside-init
self.client = Client(self.app, BaseResponse)
resp = self.client.get("/hello/123")
self.assertEqual(200, resp.status_code)
self.assertEqual([b"Hello: 123"], list(resp.response))
span_list = self.memory_exporter.get_finished_spans()
self.assertEqual(len(span_list), 1)
def tearDown(self):
super().tearDown()
with self.disable_logging():
FlaskInstrumentor().uninstrument()
import opentelemetry.ext.requests
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import ConsoleSpanExporter
from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor
from opentelemetry.ext.flask import FlaskInstrumentor
from opentelemetry.ext import jaeger
trace.set_tracer_provider(TracerProvider())
jaeger_exporter = jaeger.JaegerSpanExporter(service_name="my-flask-service",
agent_host_name="localhost",
agent_port=6831)
trace.get_tracer_provider().add_span_processor(
SimpleExportSpanProcessor(jaeger_exporter))
app = flask.Flask(__name__)
FlaskInstrumentor().instrument_app(app)
# opentelemetry.ext.http_requests.RequestsInstrumentor().instrument()
opentelemetry.ext.requests.RequestsInstrumentor().instrument()
@app.route("/")
def hello():
tracer = trace.get_tracer(__name__)
with tracer.start_as_current_span("example-request"):
requests.get("http://www.example.com")
return "hello"
app.run(debug=True, port=5100)
from opentelemetry.ext.flask import FlaskInstrumentor
FlaskInstrumentor().instrument()
from wx_explore.web.core import app, db
from wx_explore.web.api import api
app.register_blueprint(api)
from wx_explore.common.location import preload_coordinate_lookup_meta
preload_coordinate_lookup_meta()
# Clear so gunicorn forks with no (shared) engine
db.session.remove()
db.engine.dispose()
@app.before_first_request
def post_fork_init():
"""
Sentry and tracing are initialized late (after gunicorn fork) so that
any network sessions they create during init aren't shared between
processes.
"""
from wx_explore.common.logging import init_sentry
from wx_explore.common.tracing import init_tracing
init_sentry(flask=True)
init_tracing('api')
def create_app():
"""Flask application factory to create instances
of the Userservice Flask App
"""
app = Flask(__name__)
# Set up tracing and export spans to Cloud Trace
trace.set_tracer_provider(TracerProvider())
cloud_trace_exporter = CloudTraceSpanExporter()
trace.get_tracer_provider().add_span_processor(
SimpleExportSpanProcessor(cloud_trace_exporter))
set_global_httptextformat(CloudTraceFormatPropagator())
# Add Flask auto-instrumentation for tracing
FlaskInstrumentor().instrument_app(app)
# Disabling unused-variable for lines with route decorated functions
# as pylint thinks they are unused
# pylint: disable=unused-variable
@app.route('/version', methods=['GET'])
def version():
"""
Service version endpoint
"""
return app.config['VERSION'], 200
@app.route('/ready', methods=['GET'])
def readiness():
"""
Readiness probe
"""
return 'ok', 200
@app.route('/users', methods=['POST'])
def create_user():
"""Create a user record.
Fails if that username already exists.
Generates a unique accountid.
request fields:
- username
- password
- password-repeat
- firstname
- lastname
- birthday
- timezone
- address
- state
- zip
- ssn
"""
try:
app.logger.debug('Sanitizing input.')
req = {k: bleach.clean(v) for k, v in request.form.items()}
__validate_new_user(req)
# Check if user already exists
if users_db.get_user(req['username']) is not None:
raise NameError('user {} already exists'.format(
req['username']))
# Create password hash with salt
app.logger.debug("Creating password hash.")
password = req['password']
salt = bcrypt.gensalt()
passhash = bcrypt.hashpw(password.encode('utf-8'), salt)
accountid = users_db.generate_accountid()
# Create user data to be added to the database
user_data = {
'accountid': accountid,
'username': req['username'],
'passhash': passhash,
'firstname': req['firstname'],
'lastname': req['lastname'],
'birthday': req['birthday'],
'timezone': req['timezone'],
'address': req['address'],
'state': req['state'],
'zip': req['zip'],
'ssn': req['ssn'],
}
# Add user_data to database
app.logger.debug("Adding user to the database")
users_db.add_user(user_data)
app.logger.info("Successfully created user.")
except UserWarning as warn:
app.logger.error("Error creating new user: %s", str(warn))
return str(warn), 400
except NameError as err:
app.logger.error("Error creating new user: %s", str(err))
return str(err), 409
except SQLAlchemyError as err:
app.logger.error("Error creating new user: %s", str(err))
return 'failed to create user', 500
return jsonify({}), 201
def __validate_new_user(req):
app.logger.debug('validating create user request: %s', str(req))
# Check if required fields are filled
fields = (
'username',
'password',
'password-repeat',
'firstname',
'lastname',
'birthday',
'timezone',
'address',
'state',
'zip',
'ssn',
)
if any(f not in req for f in fields):
raise UserWarning('missing required field(s)')
if any(not bool(req[f] or req[f].strip()) for f in fields):
raise UserWarning('missing value for input field(s)')
# Verify username contains only 2-15 alphanumeric or underscore characters
if not re.match(r"\A[a-zA-Z0-9_]{2,15}\Z", req['username']):
raise UserWarning(
'username must contain 2-15 alphanumeric characters or underscores'
)
# Check if passwords match
if not req['password'] == req['password-repeat']:
raise UserWarning('passwords do not match')
@app.route('/login', methods=['GET'])
def login():
"""Login a user and return a JWT token
Fails if username doesn't exist or password doesn't match hash
token expiry time determined by environment variable
request fields:
- username
- password
"""
app.logger.debug('Sanitizing login input.')
username = bleach.clean(request.args.get('username'))
password = bleach.clean(request.args.get('password'))
# Get user data
try:
app.logger.debug('Getting the user data.')
user = users_db.get_user(username)
if user is None:
raise LookupError('user {} does not exist'.format(username))
# Validate the password
app.logger.debug('Validating the password.')
if not bcrypt.checkpw(password.encode('utf-8'), user['passhash']):
raise PermissionError('invalid login')
full_name = '{} {}'.format(user['firstname'], user['lastname'])
exp_time = datetime.utcnow() + timedelta(
seconds=app.config['EXPIRY_SECONDS'])
payload = {
'user': username,
'acct': user['accountid'],
'name': full_name,
'iat': datetime.utcnow(),
'exp': exp_time,
}
app.logger.debug('Creating jwt token.')
token = jwt.encode(payload,
app.config['PRIVATE_KEY'],
algorithm='RS256')
app.logger.info('Login Successful.')
return jsonify({'token': token.decode("utf-8")}), 200
except LookupError as err:
app.logger.error('Error logging in: %s', str(err))
return str(err), 404
except PermissionError as err:
app.logger.error('Error logging in: %s', str(err))
return str(err), 401
except SQLAlchemyError as err:
app.logger.error('Error logging in: %s', str(err))
return 'failed to retrieve user information', 500
@atexit.register
def _shutdown():
"""Executed when web app is terminated."""
app.logger.info("Stopping userservice.")
# Set up logger
app.logger.handlers = logging.getLogger('gunicorn.error').handlers
app.logger.setLevel(logging.getLogger('gunicorn.error').level)
app.logger.info('Starting userservice.')
app.config['VERSION'] = os.environ.get('VERSION')
app.config['EXPIRY_SECONDS'] = int(os.environ.get('TOKEN_EXPIRY_SECONDS'))
app.config['PRIVATE_KEY'] = open(os.environ.get('PRIV_KEY_PATH'),
'r').read()
app.config['PUBLIC_KEY'] = open(os.environ.get('PUB_KEY_PATH'), 'r').read()
# Configure database connection
try:
users_db = UserDb(os.environ.get("ACCOUNTS_DB_URI"), app.logger)
except OperationalError:
app.logger.critical("users_db database connection failed")
sys.exit(1)
return app
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor
# # Set up tracing and export spans to Cloud Trace.
# trace.set_tracer_provider(TracerProvider())
# CLOUD_TRACE_EXPORTER = CloudTraceSpanExporter()
# trace.get_tracer_provider().add_span_processor(
# SimpleExportSpanProcessor(CLOUD_TRACE_EXPORTER)
# )
# set_global_httptextformat(CloudTraceFormatPropagator())
APP = Flask(__name__)
# # Add tracing auto-instrumentation for Flask, jinja and requests
FlaskInstrumentor().instrument_app(APP)
RequestsInstrumentor().instrument()
Jinja2Instrumentor().instrument()
@APP.route('/version', methods=['GET'])
def version():
"""
Service version endpoint
"""
return os.environ.get('VERSION'), 200
@APP.route('/ready', methods=['GET'])
def readiness():
"""
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from opentelemetry.ext.flask import FlaskInstrumentor
_FLASK_INSTRUMENTOR = FlaskInstrumentor()
def pytest_sessionstart(session): # pylint: disable=unused-argument
_FLASK_INSTRUMENTOR.instrument()
def pytest_sessionfinish(session): # pylint: disable=unused-argument
_FLASK_INSTRUMENTOR.uninstrument()
def create_app():
"""Flask application factory to create instances
of the Connector Service Flask App
"""
app = Flask(__name__)
# Set up tracing and export spans to Cloud Trace
trace.set_tracer_provider(TracerProvider())
cloud_trace_exporter = CloudTraceSpanExporter()
trace.get_tracer_provider().add_span_processor(
SimpleExportSpanProcessor(cloud_trace_exporter))
set_global_httptextformat(CloudTraceFormatPropagator())
# Add Flask auto-instrumentation for tracing
FlaskInstrumentor().instrument_app(app)
# Disabling unused-variable for lines with route decorated functions
# as pylint thinks they are unused
# pylint: disable=unused-variable
@app.route("/version", methods=["GET"])
def version():
"""
Service version endpoint
"""
return app.config["VERSION"], 200
@app.route("/ready", methods=["GET"])
def ready():
"""Readiness probe."""
return "ok", 200
@app.route("/sync", methods=["POST"])
def sync():
"""Sync PostgreSQL metadata with Google Data Catalog
"""
auth_header = request.headers.get("Authorization")
if auth_header:
token = auth_header.split(" ")[-1]
else:
token = ""
try:
auth_payload = jwt.decode(token,
key=app.config["PUBLIC_KEY"],
algorithms="RS256")
if auth_payload is None:
raise PermissionError
logging.info("Starting sync logic.")
datacatalog_cli.PostgreSQL2DatacatalogCli().run(
_get_connector_run_args())
logging.info("Sync execution done.")
return jsonify({}), 200
except (PermissionError, jwt.exceptions.InvalidTokenError) as err:
logging.error("Error executing sync: %s", str(err))
return "authentication denied", 401
except UserWarning as warn:
logging.error("Error executing sync: %s", str(warn))
return str(warn), 400
except Exception as err:
logging.error("Error executing sync: %s", str(err))
return "failed to sync", 500
@atexit.register
def _shutdown():
"""Executed when web app is terminated."""
logging.info("Stopping contacts service.")
def _get_connector_run_args():
return [
'--datacatalog-project-id',
os.environ.get('DATACATALOG_PROJECT_ID'),
'--datacatalog-location-id',
os.environ.get('DATACATALOG_LOCATION_ID'), '--postgresql-host',
os.environ.get('POSTGRESQL_SERVER'), '--postgresql-user',
os.environ.get('POSTGRES_USER'), '--postgresql-pas',
os.environ.get('POSTGRES_PASSWORD'), '--postgresql-database',
os.environ.get('POSTGRES_DB')
]
logging_client = gcp_logging.Client()
logging_client.setup_logging(log_level=logging.INFO)
root_logger = logging.getLogger()
# use the GCP handler ONLY in order to prevent logs from getting written to STDERR
root_logger.handlers = [
handler for handler in root_logger.handlers
if isinstance(handler, (CloudLoggingHandler, ContainerEngineHandler,
AppEngineHandler))
]
logging.info("Service PostgreSQL connector created.")
# setup global variables
app.config["VERSION"] = os.environ.get("VERSION")
app.config["PUBLIC_KEY"] = open(os.environ.get("PUB_KEY_PATH"), "r").read()
return app
def create_app():
"""Flask application factory to create instances
of the Contact Service Flask App
"""
app = Flask(__name__)
# Set up tracing and export spans to Cloud Trace
trace.set_tracer_provider(TracerProvider())
cloud_trace_exporter = CloudTraceSpanExporter()
trace.get_tracer_provider().add_span_processor(
SimpleExportSpanProcessor(cloud_trace_exporter))
set_global_httptextformat(CloudTraceFormatPropagator())
# Add Flask auto-instrumentation for tracing
FlaskInstrumentor().instrument_app(app)
# Disabling unused-variable for lines with route decorated functions
# as pylint thinks they are unused
# pylint: disable=unused-variable
@app.route("/version", methods=["GET"])
def version():
"""
Service version endpoint
"""
return app.config["VERSION"], 200
@app.route("/ready", methods=["GET"])
def ready():
"""Readiness probe."""
return "ok", 200
@app.route("/contacts/<username>", methods=["GET"])
def get_contacts(username):
"""Retrieve the contacts list for the authenticated user.
This list is used for populating Payment and Deposit fields.
Return: a list of contacts
"""
auth_header = request.headers.get("Authorization")
if auth_header:
token = auth_header.split(" ")[-1]
else:
token = ""
try:
auth_payload = jwt.decode(token,
key=app.config["PUBLIC_KEY"],
algorithms="RS256")
if username != auth_payload["user"]:
raise PermissionError
contacts_list = contacts_db.get_contacts(username)
app.logger.debug("Succesfully retrieved contacts.")
return jsonify(contacts_list), 200
except (PermissionError, jwt.exceptions.InvalidTokenError) as err:
app.logger.error("Error retrieving contacts list: %s", str(err))
return "authentication denied", 401
except SQLAlchemyError as err:
app.logger.error("Error retrieving contacts list: %s", str(err))
return "failed to retrieve contacts list", 500
@app.route("/contacts/<username>", methods=["POST"])
def add_contact(username):
"""Add a new favorite account to user's contacts list
Fails if account or routing number are invalid
or if label is not alphanumeric
request fields:
- account_num
- routing_num
- label
- is_external
"""
auth_header = request.headers.get("Authorization")
if auth_header:
token = auth_header.split(" ")[-1]
else:
token = ""
try:
auth_payload = jwt.decode(token,
key=app.config["PUBLIC_KEY"],
algorithms="RS256")
if username != auth_payload["user"]:
raise PermissionError
req = {
k: (bleach.clean(v) if isinstance(v, str) else v)
for k, v in request.get_json().items()
}
_validate_new_contact(req)
_check_contact_allowed(username, auth_payload["acct"], req)
# Create contact data to be added to the database.
contact_data = {
"username": username,
"label": req["label"],
"account_num": req["account_num"],
"routing_num": req["routing_num"],
"is_external": req["is_external"],
}
# Add contact_data to database
app.logger.debug("Adding new contact to the database.")
contacts_db.add_contact(contact_data)
app.logger.info("Successfully added new contact.")
return jsonify({}), 201
except (PermissionError, jwt.exceptions.InvalidTokenError) as err:
app.logger.error("Error adding contact: %s", str(err))
return "authentication denied", 401
except UserWarning as warn:
app.logger.error("Error adding contact: %s", str(warn))
return str(warn), 400
except ValueError as err:
app.logger.error("Error adding contact: %s", str(err))
return str(err), 409
except SQLAlchemyError as err:
app.logger.error("Error adding contact: %s", str(err))
return "failed to add contact", 500
def _validate_new_contact(req):
"""Check that this new contact request has valid fields"""
app.logger.debug("validating add contact request: %s", str(req))
# Check if required fields are filled
fields = ("label", "account_num", "routing_num", "is_external")
if any(f not in req for f in fields):
raise UserWarning("missing required field(s)")
# Validate account number (must be 10 digits)
if req["account_num"] is None or not re.match(r"\A[0-9]{10}\Z",
req["account_num"]):
raise UserWarning("invalid account number")
# Validate routing number (must be 9 digits)
if req["routing_num"] is None or not re.match(r"\A[0-9]{9}\Z",
req["routing_num"]):
raise UserWarning("invalid routing number")
# Only allow external accounts to deposit
if (req["is_external"]
and req["routing_num"] == app.config["LOCAL_ROUTING"]):
raise UserWarning("invalid routing number")
# Validate label
# Must be >0 and <=30 chars, alphanumeric and spaces, can't start with space
if req["label"] is None or not re.match(
r"^[0-9a-zA-Z][0-9a-zA-Z ]{0,29}$", req["label"]):
raise UserWarning("invalid account label")
def _check_contact_allowed(username, accountid, req):
"""Check that this contact is allowed to be created"""
app.logger.debug(
"checking that this contact is allowed to be created: %s",
str(req))
# Don't allow self reference
if (req["account_num"] == accountid
and req["routing_num"] == app.config["LOCAL_ROUTING"]):
raise ValueError("may not add yourself to contacts")
# Don't allow identical contacts
for contact in contacts_db.get_contacts(username):
if (contact["account_num"] == req["account_num"]
and contact["routing_num"] == req["routing_num"]):
raise ValueError("account already exists as a contact")
if contact["label"] == req["label"]:
raise ValueError("contact already exists with that label")
@atexit.register
def _shutdown():
"""Executed when web app is terminated."""
app.logger.info("Stopping contacts service.")
# set up logger
app.logger.handlers = logging.getLogger("gunicorn.error").handlers
app.logger.setLevel(logging.getLogger("gunicorn.error").level)
app.logger.info("Starting contacts service.")
# setup global variables
app.config["VERSION"] = os.environ.get("VERSION")
app.config["LOCAL_ROUTING"] = os.environ.get("LOCAL_ROUTING_NUM")
app.config["PUBLIC_KEY"] = open(os.environ.get("PUB_KEY_PATH"), "r").read()
# Configure database connection
try:
contacts_db = ContactsDb(os.environ.get("ACCOUNTS_DB_URI"), app.logger)
except OperationalError:
app.logger.critical("database connection failed")
sys.exit(1)
return app