def edit_core_data_from_states_daily():
payload = flask.request.json
flask.current_app.logger.info(
'Received a CoreData States Daily edit request: %s' % payload)
# test input data
try:
validate_edit_data_payload(payload)
except ValueError as e:
flask.current_app.logger.error("Edit data failed validation: %s" %
str(e))
notify_slack_error(str(e), 'edit_core_data_from_states_daily')
return str(e), 400
# we construct the batch from the push context
context = payload['context']
# check that the state is set
state_to_edit = context.get('state')
if not state_to_edit:
flask.current_app.logger.error(
"No state specified in batch edit context: %s" % str(context))
notify_slack_error('No state specified in batch edit context',
'edit_core_data_from_states_daily')
return 'No state specified in batch edit context', 400
core_data = payload['coreData']
return edit_states_daily_internal(get_jwt_identity(),
context,
core_data,
state_to_edit,
publish=True)
def do_notify_webhook():
url = current_app.config['API_WEBHOOK_URL']
if not url: # nothing to do for dev environments without a url set
return
try:
response = requests.get(url)
except Exception as e:
current_app.logger.warning(
'Request to webhook %s failed returned an error: %s' %
(url, str(e)))
notify_slack_error(f"notify_webhook failed: #{str(e)}",
"do_notify_webhook")
return False
if response.status_code != 200:
# log an error but do not raise an exception
# This method would usually run *after a commit*, as a best-effort
# method, it should not fail a response to the user
current_app.logger.error(
'Request to webhook %s finished unsuccessfully: %s, the response is:\n%s'
% (url, response.status_code, response.text))
notify_slack_error(
f"notify_webhook failed (#{response.status_code}): #{response.text}",
"do_notify_webhook")
return response
def edit_core_data():
flask.current_app.logger.info('Received a CoreData edit request')
payload = flask.request.json
# test input data
try:
validate_edit_data_payload(payload)
except ValueError as e:
flask.current_app.logger.error("Edit data failed validation: %s" %
str(e))
notify_slack_error(str(e), 'edit_core_data')
return flask.jsonify(str(e)), 400
# we construct the batch from the push context
context = payload['context']
flask.current_app.logger.info('Creating new batch from context: %s' %
context)
batch = Batch(**context)
batch.user = get_jwt_identity()
batch.isRevision = True
db.session.add(batch)
db.session.flush(
) # this sets the batch ID, which we need for corresponding coreData objects
# check each core data row that the corresponding date/state already exists in published form
core_data_dicts = payload['coreData']
core_data_objects = []
for core_data_dict in core_data_dicts:
flask.current_app.logger.info('Creating new core data row: %s' %
core_data_dict)
# check that there exists at least one published row for this date/state
date = core_data_dict['date']
state = core_data_dict['state']
if not any_existing_rows(state, date):
return flask.jsonify(
"No existing published row for state %s on date %s" %
(state, date)), 400
core_data_dict['batchId'] = batch.batchId
core_data = CoreData(**core_data_dict)
db.session.add(core_data)
core_data_objects.append(core_data)
db.session.flush()
json_to_return = {
'batch': batch.to_dict(),
'coreData': [core_data.to_dict() for core_data in core_data_objects],
}
db.session.commit()
notify_slack(
f"*Pushed batch #{batch.batchId}* (type: {batch.dataEntryType}, user: {batch.shiftLead})\n"
f"{batch.batchNote}")
return flask.jsonify(json_to_return), 201
def edit_core_data():
flask.current_app.logger.info('Received a CoreData edit request')
payload = flask.request.json
# test input data
try:
validate_edit_data_payload(payload)
except ValueError as e:
flask.current_app.logger.error("Edit data failed validation: %s" %
str(e))
notify_slack_error(str(e), 'edit_core_data')
return str(e), 400
context = payload['context']
core_data = payload['coreData']
return edit_states_daily_internal(get_jwt_identity(), context, core_data)
def edit_state_metadata():
payload = flask.request.json
flask.current_app.logger.info('Received a states edit request: %s' %
payload)
# we expect the payload to contain states
if 'states' not in payload:
err = '/states/edit payload must contain "states" field'
flask.current_app.logger.error(err)
notify_slack_error(err, 'edit_state_metadata')
return err, 400
state_dicts = payload['states']
state_objects = []
for state_dict in state_dicts:
state_pk = state_dict['state']
state_obj = db.session.query(State).get(state_pk)
if state_obj is None:
err = '/states/edit payload trying to edit nonexistent state: %s' % state_pk
flask.current_app.logger.error(err)
notify_slack_error(err, 'edit_state_metadata')
return err, 400
flask.current_app.logger.info('Updating state row from info: %s' %
state_dict)
db.session.query(State).filter_by(state=state_pk).update(state_dict)
# this method of updating does not trigger validators, so validate manually
state_obj.validate_totalTestResultsFieldDbColumn(
None, state_obj.totalTestResultsFieldDbColumn)
state_objects.append(
db.session.query(State).get(state_pk)) # return updated state
db.session.flush()
# construct the JSON before committing the session, since sqlalchemy objects behave weirdly
# once the session has been committed
json_to_return = {
'states': [state.to_dict() for state in state_objects],
}
db.session.commit()
# this returns a tuple of flask response and status code: (flask.Response, int)
return flask.jsonify(json_to_return), 201
def edit_core_data_from_states_daily():
payload = flask.request.json
flask.current_app.logger.info(
'Received a CoreData States Daily edit request: %s' % payload)
# validate input data
try:
validate_edit_data_payload(payload)
except ValueError as e:
flask.current_app.logger.error("Edit data failed validation: %s" %
str(e))
notify_slack_error(str(e), 'edit_core_data_from_states_daily')
return str(e), 400
context = payload['context']
flask.current_app.logger.info('Creating new batch from context: %s' %
context)
batch = Batch(**context)
batch.user = get_jwt_identity()
batch.isRevision = True
batch.isPublished = True
batch.publishedAt = datetime.utcnow()
db.session.add(batch)
db.session.flush(
) # this sets the batch ID, which we need for corresponding coreData objects
state_to_edit = payload['context']['state']
latest_daily_data_for_state = states_daily_query(state=state_to_edit,
research=True).all()
# split up by date for easier lookup/comparison with input edit rows
key_to_date = defaultdict(dict) # state -> date -> data
for state_daily_data in latest_daily_data_for_state:
key_to_date[state_daily_data.state][
state_daily_data.date] = state_daily_data
# keep track of all our changes as we go
core_data_objects = []
changed_rows = []
new_rows = []
# check each core data row that the corresponding date/state already exists in published form
for core_data_dict in payload['coreData']:
state = core_data_dict['state']
valid, unknown = CoreData.valid_fields_checker(core_data_dict)
if not valid:
# there are no fields to add/update
flask.current_app.logger.info(
'Got row without updates, skipping. %r' % core_data_dict)
continue
# is there a date for this?
# check that there exists at least one published row for this date/state
date = CoreData.parse_str_to_date(core_data_dict['date'])
data_for_date = key_to_date.get(state, {}).get(date)
core_data_dict['batchId'] = batch.batchId
edited_core_data = None
if not data_for_date:
# this is a new row: we treat this as a changed date
# TODO: uncomment these 3 lines if we want to enforce editing only existing date rows
# error = 'Attempting to edit a nonexistent date: %s' % core_data_dict['date']
# flask.current_app.logger.error(error)
# return flask.jsonify(error), 400
flask.current_app.logger.info(
'Row for date not found, making new edit row: %s' % date)
edited_core_data = CoreData(**core_data_dict)
new_rows.append(edited_core_data)
else:
# this row already exists, check each property to see if anything changed.
changed_for_date = data_for_date.field_diffs(core_data_dict)
if changed_for_date:
changed_rows.append(changed_for_date)
edited_core_data = data_for_date.copy_with_updates(
**core_data_dict)
# if any value in the row is different, make an edit batch
if edited_core_data:
# store the changes
db.session.add(edited_core_data)
core_data_objects.append(edited_core_data)
db.session.flush()
flask.current_app.logger.info('Adding new edit row: %s' %
edited_core_data.to_dict())
else:
# there were no changes
flask.current_app.logger.info(
'All values are the same for date %s, ignoring' % date)
db.session.flush()
diffs = EditDiff(changed_rows, new_rows)
if diffs.is_empty():
# there are no changes, nothing to do
notify_slack_error(
f"*Received edit batch #{batch.batchId}*. state: {state_to_edit}. (user: {batch.shiftLead})\n"
f"{batch.batchNote} but no differences detected, data is unchanged",
"edit_states_daily")
return 'Data is unchanged: no edits detected', 400
batch.changedFields = diffs.changed_fields
batch.numRowsEdited = diffs.size()
db.session.flush()
# TODO: change consumer of this response to use the changedFields, changedDates, numRowsEdited
# from the "batch" object, then remove those keys from the JSON response
json_to_return = {
'batch': batch.to_dict(),
'changedFields': batch.changedFields,
'changedDates': diffs.changed_dates_str,
'numRowsEdited': batch.numRowsEdited,
'user': get_jwt_identity(),
'coreData': [core_data.to_dict() for core_data in core_data_objects],
}
db.session.commit()
# collect all the diffs for the edits we've made and format them for a slack notification
diffs_for_slack = diffs.plain_text_format()
notify_slack(
f"*Pushed and published {batch.dataEntryType} batch #{batch.batchId}*. state: {state_to_edit}. (user: {batch.shiftLead})\n"
f"{batch.batchNote}", diffs_for_slack)
return flask.jsonify(json_to_return), 201
def post_core_data_json(payload):
# test the input data
try:
validate_core_data_payload(payload)
except ValueError as e:
flask.current_app.logger.error('Data post failed: %s' % str(e))
notify_slack_error(str(e), 'post_core_data_json')
return str(e), 400
# we construct the batch from the push context
context = payload['context']
flask.current_app.logger.info('Creating new batch from context: %s' %
context)
batch = Batch(**context)
batch.user = get_jwt_identity()
db.session.add(batch)
db.session.flush(
) # this sets the batch ID, which we need for corresponding coreData objects
# add states if exist
state_dicts = payload.get('states', [])
state_objects = []
for state_dict in state_dicts:
state_pk = state_dict['state']
state_obj = db.session.query(State).get(state_pk)
if state_obj is not None:
flask.current_app.logger.info('Updating state row from info: %s' %
state_dict)
db.session.query(State).filter_by(
state=state_pk).update(state_dict)
# this method of updating does not trigger validators, so validate manually
state_obj.validate_totalTestResultsFieldDbColumn(
None, state_obj.totalTestResultsFieldDbColumn)
state_objects.append(state_obj) # return updated state
else:
flask.current_app.logger.info(
'Creating new state row from info: %s' % state_dict)
state = State(**state_dict)
db.session.add(state)
state_objects.append(state)
db.session.flush()
# add all core data rows
core_data_dicts = payload['coreData']
core_data_objects = []
for core_data_dict in core_data_dicts:
flask.current_app.logger.info('Creating new core data row: %s' %
core_data_dict)
core_data_dict['batchId'] = batch.batchId
core_data = CoreData(**core_data_dict)
db.session.add(core_data)
core_data_objects.append(core_data)
db.session.flush()
# construct the JSON before committing the session, since sqlalchemy objects behave weirdly
# once the session has been committed
json_to_return = {
'batch': batch.to_dict(),
'coreData': [core_data.to_dict() for core_data in core_data_objects],
'states': [state.to_dict() for state in state_objects],
}
db.session.commit()
# this returns a tuple of flask response and status code: (flask.Response, int)
return flask.jsonify(json_to_return), 201
def edit_core_data_from_states_daily():
payload = flask.request.json
flask.current_app.logger.info(
'Received a CoreData States Daily edit request: %s' % payload)
# test input data
try:
validate_edit_data_payload(payload)
except ValueError as e:
flask.current_app.logger.error("Edit data failed validation: %s" %
str(e))
notify_slack_error(str(e), 'edit_core_data_from_states_daily')
return flask.jsonify(str(e)), 400
# we construct the batch from the push context
context = payload['context']
# check that the state is set
state_to_edit = context.get('state')
if not state_to_edit:
flask.current_app.logger.error(
"No state specified in batch edit context: %s" % str(e))
notify_slack_error('No state specified in batch edit context',
'edit_core_data_from_states_daily')
return flask.jsonify('No state specified in batch edit context'), 400
flask.current_app.logger.info('Creating new batch from context: %s' %
context)
batch = Batch(**context)
batch.user = get_jwt_identity()
batch.isRevision = True
batch.isPublished = True # edit batches are published by default
batch.publishedAt = datetime.utcnow()
db.session.add(batch)
db.session.flush(
) # this sets the batch ID, which we need for corresponding coreData objects
latest_daily_data_for_state = states_daily_query(state=state_to_edit).all()
# split up by date for easier lookup/comparison with input edit rows
date_to_data = {}
for state_daily_data in latest_daily_data_for_state:
date_to_data[state_daily_data.date] = state_daily_data
# check each core data row that the corresponding date/state already exists in published form
core_data_objects = []
changed_fields = set()
changed_dates = set()
for core_data_dict in payload['coreData']:
# this state has to be identical to the state from the context
state = core_data_dict['state']
if state != state_to_edit:
error = 'Context state %s does not match JSON data state %s' % (
state_to_edit, state)
flask.current_app.logger.error(error)
notify_slack_error(error, 'edit_core_data_from_states_daily')
return flask.jsonify(error), 400
# is there a date for this?
# check that there exists at least one published row for this date/state
date = CoreData.parse_str_to_date(core_data_dict['date'])
data_for_date = date_to_data.get(date)
changed_fields_for_date = set()
is_edited = False
# make a new CoreData object, which we will add if we determine it represents an edited row
core_data_dict['batchId'] = batch.batchId
edited_core_data = CoreData(**core_data_dict)
if not data_for_date:
# this is a new row: we treat this as a changed date
# TODO: uncomment these 3 lines if we want to enforce editing only existing date rows
# error = 'Attempting to edit a nonexistent date: %s' % core_data_dict['date']
# flask.current_app.logger.error(error)
# return flask.jsonify(error), 400
flask.current_app.logger.info(
'Row for date not found, making new edit row: %s' % date)
is_edited = True
else:
# this row already exists, but check each value to see if anything changed. Easiest way
# to do this is to make a new CoreData and compare it to the existing one
for field in CoreData.__table__.columns.keys():
# we expect batch IDs to be different, skip comparing those
if field == 'batchId':
continue
# for any other field, compare away
if getattr(data_for_date, field) != getattr(
edited_core_data, field):
changed_fields_for_date.add(field)
is_edited = True
# if any value in the row is different, make an edit batch
if is_edited:
db.session.add(edited_core_data)
core_data_objects.append(edited_core_data)
flask.current_app.logger.info('Adding new edit row: %s' %
edited_core_data.to_dict())
changed_fields.update(changed_fields_for_date)
changed_dates.add(date)
else:
flask.current_app.logger.info(
'All values are the same for date %s, ignoring' % date)
db.session.flush()
# which dates got changed?
start = sorted(changed_dates)[0].strftime('%-m/%-d/%y')
end = sorted(changed_dates)[-1].strftime('%-m/%-d/%y')
changed_dates_str = start if start == end else '%s - %s' % (start, end)
json_to_return = {
'batch': batch.to_dict(),
'changedFields': list(changed_fields),
'changedDates': changed_dates_str,
'numRowsEdited': len(changed_dates),
'user': get_jwt_identity(),
'coreData': [core_data.to_dict() for core_data in core_data_objects],
}
db.session.commit()
notify_slack(
f"*Pushed and published edit batch #{batch.batchId}*. state: {state_to_edit}. (user: {batch.shiftLead})\n"
f"{batch.batchNote}")
return flask.jsonify(json_to_return), 201
def edit_states_daily_internal(user,
context,
core_data,
state_to_edit=None,
publish=False):
flask.current_app.logger.info('Creating new batch from context: %s' %
context)
batch = Batch(**context)
batch.user = user
batch.isRevision = True
batch.isPublished = publish
if publish:
batch.publishedAt = datetime.utcnow()
db.session.add(batch)
db.session.flush(
) # this sets the batch ID, which we need for corresponding coreData objects
latest_daily_data_for_state = states_daily_query(state=state_to_edit).all()
# split up by date for easier lookup/comparison with input edit rows
date_to_data = {}
for state_daily_data in latest_daily_data_for_state:
date_to_data[state_daily_data.date] = state_daily_data
# keep track of all our changes as we go
core_data_objects = []
changed_rows = []
new_rows = []
# check each core data row that the corresponding date/state already exists in published form
for core_data_dict in core_data:
# this state has to be identical to the state from the context
state = core_data_dict['state']
if state_to_edit and state != state_to_edit:
error = 'Context state %s does not match JSON data state %s' % (
state_to_edit, state)
flask.current_app.logger.error(error)
notify_slack_error(error, 'edit_core_data_from_states_daily')
return error, 400
valid, unknown = CoreData.valid_fields_checker(core_data_dict)
if not valid:
# there are no fields to add/update
flask.current_app.logger.info(
'Got row without updates, skipping. %r' % core_data_dict)
continue
if unknown:
# report unknown fields, we won't fail the request, but should at least log
flask.current_app.logger.warning(
'Got row with unknown field updates: %s. %r' %
(unknown, core_data_dict))
# is there a date for this?
# check that there exists at least one published row for this date/state
date = CoreData.parse_str_to_date(core_data_dict['date'])
data_for_date = date_to_data.get(date)
core_data_dict['batchId'] = batch.batchId
edited_core_data = None
if not data_for_date:
# this is a new row: we treat this as a changed date
# TODO: uncomment these 3 lines if we want to enforce editing only existing date rows
# error = 'Attempting to edit a nonexistent date: %s' % core_data_dict['date']
# flask.current_app.logger.error(error)
# return flask.jsonify(error), 400
flask.current_app.logger.info(
'Row for date not found, making new edit row: %s' % date)
edited_core_data = CoreData(**core_data_dict)
new_rows.append(edited_core_data)
else:
# this row already exists, but check each value to see if anything changed. Easiest way
changed_for_date = data_for_date.field_diffs(core_data_dict)
if changed_for_date:
changed_rows.append(changed_for_date)
edited_core_data = data_for_date.copy_with_updates(
**core_data_dict)
# if any value in the row is different, make an edit batch
if edited_core_data:
# store the changes
db.session.add(edited_core_data)
core_data_objects.append(edited_core_data)
db.session.flush()
flask.current_app.logger.info('Adding new edit row: %s' %
edited_core_data.to_dict())
else:
# there were no changes
flask.current_app.logger.info(
'All values are the same for date %s, ignoring' % date)
db.session.flush()
diffs = EditDiff(changed_rows, new_rows)
if diffs.is_empty():
# there are no changes, nothing to do
notify_slack_error(
f"*Received edit batch #{batch.batchId}*. state: {state_to_edit}. (user: {batch.shiftLead})\n"
f"{batch.batchNote} but no differences detected, data is unchanged",
"edit_states_daily")
return 'Data is unchanged: no edits detected', 400
batch.changedFields = diffs.changed_fields
batch.numRowsEdited = diffs.size()
db.session.flush()
# TODO: change consumer of this response to use the changedFields, changedDates, numRowsEdited
# from the "batch" object, then remove those keys from the JSON response
json_to_return = {
'batch': batch.to_dict(),
'changedFields': batch.changedFields,
'changedDates': diffs.changed_dates_str,
'numRowsEdited': batch.numRowsEdited,
'user': get_jwt_identity(),
'coreData': [core_data.to_dict() for core_data in core_data_objects],
}
db.session.commit()
# collect all the diffs for the edits we've made and format them for a slack notification
diffs_for_slack = diffs.plain_text_format()
notify_slack(
f"*Pushed and published edit batch #{batch.batchId}*. state: {state_to_edit}. (user: {batch.shiftLead})\n"
f"{batch.batchNote}", diffs_for_slack)
return flask.jsonify(json_to_return), 201