def job_context(job_id, is_local=True):
"""Common context for files D1, D2, E, and F generation. Handles marking the job finished and/or failed"""
# Flask context ensures we have access to global.g
with Flask(__name__).app_context():
sess = GlobalDB.db().session
try:
yield sess
logger.info({
'message': 'Marking job {} as finished'.format(job_id),
'message_type': 'BrokerInfo',
'job_id': job_id
})
mark_job_status(job_id, "finished")
except Exception as e:
# logger.exception() automatically adds traceback info
logger.exception({
'message':
'Marking job {} as failed'.format(job_id),
'message_type':
'BrokerException',
'job_id':
job_id,
'exception':
str(e)
})
job = sess.query(Job).filter_by(job_id=job_id).one_or_none()
if job:
# mark job as failed
job.error_message = str(e)
sess.commit()
mark_job_status(job_id, "failed")
# ensure FileRequest from failed job is not cached
file_request = sess.query(FileRequest).filter_by(
job_id=job_id).one_or_none()
if file_request and file_request.is_cached_file:
file_request.is_cached_file = False
sess.commit()
finally:
file_request = sess.query(FileRequest).filter_by(
job_id=job_id).one_or_none()
if file_request and file_request.is_cached_file:
# copy job data to all child FileRequests
child_requests = sess.query(FileRequest).filter_by(
parent_job_id=job_id).all()
file_type = FILE_TYPE_DICT_LETTER[
file_request.job.file_type_id]
for child in child_requests:
copy_parent_file_request_data(sess, child.job,
file_request.job, file_type,
is_local)
GlobalDB.close()
def full_database_setup():
"""Sets up a clean database based on the model metadata. It also
calculates the FK relationships between tables so we can delete them in
order. It yields a tuple the _DB and ordered list of tables."""
rand_id = str(randint(1, 9999))
config = dataactcore.config.CONFIG_DB
config['db_name'] = 'unittest{}_data_broker'.format(rand_id)
dataactcore.config.CONFIG_DB = config
create_database(config['db_name'])
db = GlobalDB.db()
run_migrations()
creation_order = baseModel.Base.metadata.sorted_tables
yield (db, list(reversed(creation_order))) # drop order
GlobalDB.close()
drop_database(config['db_name'])
def full_database_setup():
"""Sets up a clean database based on the model metadata. It also
calculates the FK relationships between tables so we can delete them in
order. It yields a tuple the _DB and ordered list of tables."""
rand_id = str(randint(1, 9999))
config = dataactcore.config.CONFIG_DB
config['db_name'] = 'unittest{}_data_broker'.format(rand_id)
dataactcore.config.CONFIG_DB = config
create_database(config['db_name'])
db = GlobalDB.db()
run_migrations()
creation_order = baseModel.Base.metadata.sorted_tables
yield (db, list(reversed(creation_order))) # drop order
GlobalDB.close()
drop_database(config['db_name'])
def job_context(job_id):
"""Common context for file E and F generation. Handles marking the job
finished and/or failed"""
# Flask context ensures we have access to global.g
with Flask(__name__).app_context():
sess = GlobalDB.db().session
try:
yield sess
logger.debug('Marking job as finished')
mark_job_status(job_id, "finished")
except Exception as e:
# logger.exception() automatically adds traceback info
logger.exception('Job %s failed', job_id)
job = sess.query(Job).filter_by(job_id=job_id).one_or_none()
if job:
job.error_message = str(e)
sess.commit()
mark_job_status(job_id, "failed")
finally:
GlobalDB.close()
def teardown_appcontext(exception):
GlobalDB.close()
def tearDownClass(cls):
"""Tear down class-level resources."""
GlobalDB.close()
drop_database(CONFIG_DB['db_name'])
def job_context(job_id, is_local=True):
""" Common context for files D1, D2, E, and F generation. Handles marking the job finished and/or failed
Args:
job_id: the ID of the submission job
is_local: a boolean indicating whether this is being run in a local environment or not
Yields:
The current DB session
"""
# Flask context ensures we have access to global.g
with Flask(__name__).app_context():
sess, job = retrieve_job_context_data(job_id)
try:
yield sess, job
if not job.from_cached:
# only mark completed jobs as done
logger.info({
'message':
'Marking job {} as finished'.format(job.job_id),
'job_id':
job.job_id,
'message_type':
'ValidatorInfo'
})
mark_job_status(job.job_id, "finished")
except Exception as e:
# logger.exception() automatically adds traceback info
logger.exception({
'message':
'Marking job {} as failed'.format(job.job_id),
'job_id':
job.job_id,
'message_type':
'ValidatorException',
'exception':
str(e)
})
# mark job as failed
job.error_message = str(e)
mark_job_status(job.job_id, "failed")
# ensure FileRequest from failed job is not cached
file_request = sess.query(FileRequest).filter_by(
job_id=job.job_id).one_or_none()
if file_request and file_request.is_cached_file:
file_request.is_cached_file = False
sess.commit()
finally:
file_request = sess.query(FileRequest).filter_by(
job_id=job.job_id).one_or_none()
if file_request and file_request.is_cached_file:
# copy job data to all child FileRequests
child_requests = sess.query(FileRequest).filter_by(
parent_job_id=job.job_id).all()
if len(child_requests) > 0:
logger.info({
'message':
'Copying file data from job {} to its children'.format(
job.job_id),
'message_type':
'ValidatorInfo',
'job_id':
job.job_id
})
for child in child_requests:
copy_parent_file_request_data(sess, child.job, job,
is_local)
GlobalDB.close()
def run_app():
"""Run the application."""
app = create_app()
# This is for DataDog (Do Not Delete)
if USE_DATADOG:
TraceMiddleware(app,
tracer,
service="broker-dd",
distributed_tracing=False)
with app.app_context():
current_app.debug = CONFIG_SERVICES['debug']
local = CONFIG_BROKER['local']
g.is_local = local
error_report_path = CONFIG_SERVICES['error_report_path']
current_app.config.from_object(__name__)
# Create connection to job tracker database
sess = GlobalDB.db().session
# Future: Override config w/ environment variable, if set
current_app.config.from_envvar('VALIDATOR_SETTINGS', silent=True)
queue = sqs_queue()
messages = []
logger.info("Starting SQS polling")
while True:
# Set current_message to None before every loop to ensure it's never set to the previous message
current_message = None
try:
# Grabs one (or more) messages from the queue
messages = queue.receive_messages(
WaitTimeSeconds=10, MessageAttributeNames=['All'])
for message in messages:
logger.info("Message received: %s", message.body)
# Retrieve the job_id from the message body
current_message = message
g.job_id = message.body
mark_job_status(g.job_id, "ready")
# Get the job
job = sess.query(Job).filter_by(
job_id=g.job_id).one_or_none()
if job is None:
validation_error_type = ValidationError.jobError
write_file_error(g.job_id, None, validation_error_type)
raise ResponseException(
'Job ID {} not found in database'.format(g.job_id),
StatusCode.CLIENT_ERROR, None,
validation_error_type)
# We have two major functionalities in the Validator: validation and file generation
if (not job.file_type or job.file_type.letter_name
in ['A', 'B', 'C', 'FABS'] or job.job_type.name !=
'file_upload') and job.submission_id:
# Run validations
validation_manager = ValidationManager(
local, error_report_path)
validation_manager.validate_job(job.job_id)
else:
# Retrieve the agency code data from the message attributes
msg_attr = current_message.message_attributes
agency_code = msg_attr['agency_code']['StringValue'] if msg_attr and \
msg_attr.get('agency_code') else None
agency_type = msg_attr['agency_type']['StringValue'] if msg_attr and \
msg_attr.get('agency_type') else None
file_generation_manager = FileGenerationManager(
job, agency_code, agency_type, local)
file_generation_manager.generate_from_job()
sess.commit()
sess.refresh(job)
# Delete from SQS once processed
message.delete()
except ResponseException as e:
# Handle exceptions explicitly raised during validation.
logger.error(traceback.format_exc())
job = get_current_job()
if job:
if job.filename is not None:
# Insert file-level error info to the database
write_file_error(job.job_id, job.filename, e.errorType,
e.extraInfo)
if e.errorType != ValidationError.jobError:
# Job passed prerequisites for validation but an error happened somewhere: mark job as 'invalid'
mark_job_status(job.job_id, 'invalid')
if current_message:
if e.errorType in [
ValidationError.rowCountError,
ValidationError.headerError,
ValidationError.fileTypeError
]:
current_message.delete()
except Exception as e:
# Handle uncaught exceptions in validation process.
logger.error(traceback.format_exc())
# csv-specific errors get a different job status and response code
if isinstance(e, ValueError) or isinstance(
e, csv.Error) or isinstance(e, UnicodeDecodeError):
job_status = 'invalid'
else:
job_status = 'failed'
job = get_current_job()
if job:
if job.filename is not None:
error_type = ValidationError.unknownError
if isinstance(e, UnicodeDecodeError):
error_type = ValidationError.encodingError
# TODO Is this really the only case where the message should be deleted?
if current_message:
current_message.delete()
write_file_error(job.job_id, job.filename, error_type)
mark_job_status(job.job_id, job_status)
finally:
GlobalDB.close()
# Set visibility to 0 so that another attempt can be made to process in SQS immediately,
# instead of waiting for the timeout window to expire
for message in messages:
try:
message.change_visibility(VisibilityTimeout=0)
except ClientError:
# Deleted messages will throw errors, which is fine because they are handled
pass
def teardown_appcontext(exception):
GlobalDB.close()
def run_app():
"""Run the application."""
app = Flask(__name__)
with app.app_context():
current_app.debug = CONFIG_SERVICES['debug']
local = CONFIG_BROKER['local']
g.is_local = local
error_report_path = CONFIG_SERVICES['error_report_path']
current_app.config.from_object(__name__)
# Future: Override config w/ environment variable, if set
current_app.config.from_envvar('VALIDATOR_SETTINGS', silent=True)
queue = sqs_queue()
logger.info("Starting SQS polling")
current_message = None
while True:
try:
# Grabs one (or more) messages from the queue
messages = queue.receive_messages(WaitTimeSeconds=10)
for message in messages:
logger.info("Message received: %s", message.body)
current_message = message
GlobalDB.db()
g.job_id = message.body
mark_job_status(g.job_id, "ready")
validation_manager = ValidationManager(local, error_report_path)
validation_manager.validate_job(g.job_id)
# delete from SQS once processed
message.delete()
except ResponseException as e:
# Handle exceptions explicitly raised during validation.
logger.error(str(e))
job = get_current_job()
if job:
if job.filename is not None:
# insert file-level error info to the database
write_file_error(job.job_id, job.filename, e.errorType, e.extraInfo)
if e.errorType != ValidationError.jobError:
# job pass prerequisites for validation, but an error
# happened somewhere. mark job as 'invalid'
mark_job_status(job.job_id, 'invalid')
if current_message:
if e.errorType in [ValidationError.rowCountError, ValidationError.headerError,
ValidationError.fileTypeError]:
current_message.delete()
except Exception as e:
# Handle uncaught exceptions in validation process.
logger.error(str(e))
# csv-specific errors get a different job status and response code
if isinstance(e, ValueError) or isinstance(e, csv.Error) or isinstance(e, UnicodeDecodeError):
job_status = 'invalid'
else:
job_status = 'failed'
job = get_current_job()
if job:
if job.filename is not None:
error_type = ValidationError.unknownError
if isinstance(e, UnicodeDecodeError):
error_type = ValidationError.encodingError
# TODO Is this really the only case where the message should be deleted?
if current_message:
current_message.delete()
write_file_error(job.job_id, job.filename, error_type)
mark_job_status(job.job_id, job_status)
finally:
GlobalDB.close()
# Set visibility to 0 so that another attempt can be made to process in SQS immediately,
# instead of waiting for the timeout window to expire
for message in messages:
message.change_visibility(VisibilityTimeout=0)