def extract_pmi_details(request_id, data_selection_condition,
request_completed_attribute, data_completed_attribute):
current_app.logger.info(f'extract_pmi_details (request_id={request_id})')
try:
dr = DemographicsRequest.query.get(request_id)
if dr is None:
raise Exception('request not found')
drd = DemographicsRequestData.query.filter_by(
demographics_request_id=request_id).filter(
data_selection_condition).first()
if drd is None:
current_app.logger.info(
f'extract_pmi_details (request_id={request_id}): Done')
setattr(dr, request_completed_attribute, datetime.utcnow())
db.session.add(dr)
else:
if not drd.has_error and drd.pmi_data is None:
get_pmi_details(drd)
setattr(drd, data_completed_attribute, datetime.utcnow())
db.session.add(drd)
db.session.commit()
schedule_lookup_tasks(request_id)
except Exception as e:
db.session.rollback()
log_exception(e)
save_demographics_error(request_id, e)
def produce_demographics_result(demographics_request_id):
current_app.logger.info(
f'produce_demographics_result (demographics_request_id={demographics_request_id})'
)
try:
dr = DemographicsRequest.query.get(demographics_request_id)
current_app.logger.info(
f'produce_demographics_result: Creating result')
dr.create_result()
dr.result_created_datetime = datetime.utcnow()
db.session.add(dr)
email(
subject='Identity Demographics Request Complete',
recipients=[dr.owner.email],
message='Your demographics request {} is complete.'.format(
url_for('ui.demographics', _external=True), ),
html=render_template('email/request_complete.html', request=dr),
)
db.session.commit()
except Exception as e:
current_app.logger.info('produce_demographics_result: Rolling Back')
db.session.rollback()
log_exception(e)
save_demographics_error(demographics_request_id, e)
def process_demographics_request_data(request_id):
current_app.logger.info(
f'process_demographics_request_data: request_id={request_id})')
try:
dr = DemographicsRequest.query.get(request_id)
if dr is None:
raise Exception('request not found')
drd = DemographicsRequestData.query.filter_by(
demographics_request_id=request_id).filter(
DemographicsRequestData.processed_datetime.is_(None)).first()
if drd is None:
dr.lookup_completed_datetime = datetime.utcnow()
db.session.add(dr)
else:
# if not drd.has_error:
spine_lookup(drd)
drd.processed_datetime = datetime.utcnow()
db.session.add(drd)
db.session.commit()
schedule_lookup_tasks(request_id)
except Exception as e:
log_exception(e)
save_demographics_error(request_id, e)
def run(self):
for instance in RedcapInstance.query.all():
try:
self._load_instance_projects(instance)
except Exception as e:
log_exception(e)
db.session.commit()
def get_pmi_details(drd):
current_app.logger.info(f'get_pmi_details (Data request Data={drd.id})')
try:
error, v_nhs_number = convert_nhs_number(drd.nhs_number)
if error is not None:
drd.messages.append(
DemographicsRequestDataMessage(
type='warning',
source='pmi_details',
scope='nhs_number',
message=error,
))
error, v_s_number = convert_uhl_system_number(drd.uhl_system_number)
if error is not None:
drd.messages.append(
DemographicsRequestDataMessage(
type='warning',
source='pmi_details',
scope='uhl_system_number',
message=error,
))
pmi = get_pmi(nhs_number=v_nhs_number, uhl_system_number=v_s_number)
if pmi is not None:
pmi_details = DemographicsRequestPmiData(**pmi._asdict())
pmi_details.demographics_request_data_id = drd.id
db.session.add(pmi_details)
except PmiException as e:
drd.messages.append(
DemographicsRequestDataMessage(
type='error',
source='pmi_details',
scope='pmi_details',
message=e.message,
))
except Exception as e:
log_exception(e)
drd.messages.append(
DemographicsRequestDataMessage(
type='error',
source='pmi_details',
scope='pmi_details',
message=traceback.format_exc(),
))
def run(self):
id_cache = {}
for pid in ParticipantImportDefinition.query.filter(
ParticipantImportDefinition.active == True).all():
try:
if pid.ecrf_source.has_new_data(pid.latest_timestamp):
self._load_participants(pid, id_cache)
db.session.commit()
except Exception as e:
log_exception(e)
self._merge_participants()
def do_lookup_tasks(demographics_request_id):
current_app.logger.info(
f'schedule_lookup_tasks (demographics_request_id={demographics_request_id})'
)
try:
dr = DemographicsRequest.query.get(demographics_request_id)
if dr is None:
raise Exception(
'Request id={} not found'.format(demographics_request_id))
if dr.paused or dr.deleted or dr.result_created or dr.in_error:
raise ScheduleException(
f'Request id={demographics_request_id} scheduled when status is "{dr.status}""'
)
current_app.logger.info(
f'Scheduling demographics_request_id={demographics_request_id} with status "{dr.status}"'
)
if not dr.data_extracted:
extract_data.delay(demographics_request_id)
elif not dr.pmi_data_pre_completed and not dr.skip_pmi:
extract_pre_pmi_details.delay(demographics_request_id)
elif not dr.lookup_completed:
process_demographics_request_data.delay(demographics_request_id)
elif not dr.pmi_data_post_completed and not dr.skip_pmi:
extract_post_pmi_details.delay(demographics_request_id)
elif not dr.result_created_datetime:
produce_demographics_result.delay(demographics_request_id)
db.session.add(dr)
db.session.commit()
except ScheduleException as sde:
current_app.logger.warning(sde)
except Exception as e:
log_exception(e)
save_demographics_error(demographics_request_id, e)
def demographics():
form = DemographicsLookupForm()
if current_user.is_admin:
search_form = DemographicsAdminSearchForm(formdata=request.args)
submitter_ids = DemographicsRequest.query.with_entities(
DemographicsRequest.owner_user_id.distinct()).filter(
DemographicsRequest.owner_user_id != current_user.id).subquery(
)
submitters = sorted(User.query.filter(
User.id.in_(submitter_ids)).all(),
key=lambda u: u.full_name)
search_form.owner_user_id.choices = [
(current_user.id, current_user.full_name)
] + [(u.id, u.full_name) for u in submitters] + [(0, 'All')]
else:
search_form = DemographicsSearchForm(formdata=request.args)
q = DemographicsRequest.query
if search_form.search.data:
q = q.filter(
DemographicsRequest.filename.like('%{}%'.format(
search_form.search.data)))
if not search_form.show_deleted.data:
q = q.filter(DemographicsRequest.deleted_datetime.is_(None))
if not search_form.show_downloaded.data:
q = q.filter(DemographicsRequest.result_downloaded_datetime.is_(None))
if current_user.is_admin:
owner_id = search_form.owner_user_id.data or current_user.id
if owner_id > 0:
q = q.filter(DemographicsRequest.owner_user_id == owner_id)
else:
q = q.filter(DemographicsRequest.owner == current_user)
demographics_requests = q.order_by(
DemographicsRequest.created_datetime.desc()).paginate(
page=search_form.page.data, per_page=5, error_out=False)
if form.validate_on_submit():
_, file_extension = os.path.splitext(form.upload.data.filename)
if file_extension == '.csv':
d = DemographicsRequestCsv(
owner=current_user,
last_updated_by_user=current_user,
filename=form.upload.data.filename,
skip_pmi=form.skip_pmi.data,
)
elif file_extension == '.xlsx':
d = DemographicsRequestXlsx(
owner=current_user,
last_updated_by_user=current_user,
filename=form.upload.data.filename,
skip_pmi=form.skip_pmi.data,
)
elif file_extension == '.xls':
d = DemographicsRequestExcel97(
owner=current_user,
last_updated_by_user=current_user,
filename=form.upload.data.filename,
skip_pmi=form.skip_pmi.data,
)
db.session.add(d)
db.session.flush()
os.makedirs(os.path.dirname(d.filepath), exist_ok=True)
form.upload.data.save(d.filepath)
try:
for name in d.get_column_names():
c = DemographicsRequestColumn(
demographics_request=d,
name=name,
last_updated_by_user=current_user,
)
db.session.add(c)
db.session.commit()
except Exception as e:
db.session.rollback()
os.unlink(d.filepath)
log_exception(e)
flash('File contents are invalid', 'error')
return redirect(url_for('ui.demographics'))
return redirect(url_for('ui.demographics_define_columns', id=d.id))
return render_template("ui/demographics/index.html",
form=form,
demographics_requests=demographics_requests,
search_form=search_form)
def extract_data(request_id):
current_app.logger.info(f'extract_data (request_id={request_id})')
try:
dr = DemographicsRequest.query.get(request_id)
if dr is None:
raise Exception('request not found')
cd = dr.column_definition
if len(dr.data) > 0:
raise Exception(
'Attempting to extract data from DemographicsRequest ("{}") '
'that has already had data extracted.'.format(request_id))
if cd is None:
raise Exception(
'Attempting to extract data from DemographicsRequest ("{}") '
'that did not have a column definition.'.format(request_id))
for i, r in enumerate(dr.iter_rows()):
uhl_system_number = get_column_value(r,
cd.uhl_system_number_column)
nhs_number = get_column_value(r, cd.nhs_number_column)
family_name = get_column_value(r, cd.family_name_column)
given_name = get_column_value(r, cd.given_name_column)
gender = get_column_value(r, cd.gender_column)
dob = get_column_value(r, cd.dob_column)
postcode = get_column_value(r, cd.postcode_column)
if any([
uhl_system_number, nhs_number, family_name, given_name,
gender, dob, postcode
]):
d = DemographicsRequestData(
demographics_request=dr,
row_number=i,
uhl_system_number=uhl_system_number,
nhs_number=nhs_number,
family_name=family_name,
given_name=given_name,
gender=gender,
dob=dob,
postcode=postcode,
)
current_app.logger.info(f'Saving extracting data={d}')
dr.data.append(d)
else:
current_app.logger.info(f'Skipping empty data')
dr.data_extracted_datetime = datetime.utcnow()
db.session.add(dr)
db.session.commit()
schedule_lookup_tasks(request_id)
except Exception as e:
db.session.rollback()
log_exception(e)
save_demographics_error(request_id, e)