def test_failed(self):
""" Verify return false when email is not sent"""
ese = self.email_settings
self.assertFalse(redi_email.send_email_redcap_connection_error(ese))
self.assertFalse(redi_email.send_email_input_data_unchanged(ese))
def generate_output(person_tree, redcap_settings, email_settings,
data_repository):
# redi.configure_logger(system_log_file_full_path)
# the global dictionary to be returned
report_data = {'errors': []}
"""
For each person we keep a count for each form type:
subject_details = array(
'person_A' => array('form_1': 1, 'form_2': 10, ...
'person_B' => array('form_1': 1, 'form_2': 10, ...
...
)
"""
subject_details = {}
# For each form type we keep a global count
form_details = {}
# count how many `person` elements are parsed
person_count = 0
root = person_tree.getroot()
persons = root.xpath('//person')
try:
# Communication with redcap
redcapClientObject = redcapClient(redcap_settings['redcap_uri'],
redcap_settings['token'],
redcap_settings['verify_ssl'])
except RequestException:
redi_email.send_email_redcap_connection_error(email_settings)
quit()
rate_limiter_value_in_redcap = float(
redcap_settings['rate_limiter_value_in_redcap'])
ideal_time_per_request = 60 / float(rate_limiter_value_in_redcap)
time_stamp_after_request = 0
# main loop for each person
for person in persons:
time_begin = datetime.datetime.now()
person_count += 1
study_id = (person.xpath('study_id') or [None])[0]
if study_id is None:
raise Exception('Expected a valid value for study_id')
# count how many csv fragments are created per person
event_count = 0
logger.info('Start sending data for study_id: %s' % study_id.text)
forms = person.xpath('./all_form_events/form')
# loop through the forms of one person
for form in forms:
form_name = form.xpath('name')[0].text
form_key = 'Total_' + form_name + '_Forms'
study_id_key = study_id.text
# init dictionary for a new person in (study_id)
if study_id_key not in subject_details:
subject_details[study_id_key] = {}
if not form_key in subject_details[study_id_key]:
subject_details[study_id_key][form_key] = 0
if form_key not in form_details:
form_details[form_key] = 0
logger.debug('parsing study_id ' + study_id.text + ' form: ' +
form_name)
# loop through the events of one form
for event in form.xpath('event'):
event_status = event.findtext('status')
if event_status == 'sent':
continue
event_count += 1
try:
import_dict = {
redcapClientObject.project.def_field: study_id.text
}
import_dict = create_import_data_json(import_dict, event)
json_data_dict = import_dict['json_data']
contains_data = import_dict['contains_data']
time_lapse_since_last_request = time.time(
) - time_stamp_after_request
sleepTime = max(
ideal_time_per_request - time_lapse_since_last_request,
0)
# print 'Sleep for: %s seconds' % sleepTime
time.sleep(sleepTime)
if (0 == event_count % 50):
logger.info('Requests sent: %s' % (event_count))
# to speedup testing uncomment the following line
# if (0 == event_count % 2) : continue
try:
found_error = False
response = redcapClientObject.send_data_to_redcap(
[json_data_dict], overwrite=True)
status = event.find('status')
if status is not None:
status.text = 'sent'
else:
status_element = etree.Element("status")
status_element.text = 'sent'
event.append(status_element)
data_repository.store(person_tree)
except RedcapError as e:
found_error = handle_errors_in_redcap_xml_response(
e.message, report_data)
time_stamp_after_request = time.time()
if contains_data:
if not found_error:
# if no errors encountered update event counters
subject_details[study_id_key][form_key] += 1
form_details[form_key] += 1
except Exception as e:
logger.error(e.message)
raise
time_end = datetime.datetime.now()
logger.info("Total execution time for study_id %s was %s" %
(study_id_key, (time_end - time_begin)))
logger.info("Total REDCap requests sent: %s \n" % (event_count))
report_data.update({
'total_subjects': person_count,
'form_details': form_details,
'subject_details': subject_details,
'errors': report_data['errors']
})
logger.debug('report_data ' + repr(report_data))
return report_data
def test_success(self):
""" Verify return true when email is sent"""
ese = self.email_settings
self.assertTrue(redi_email.send_email_redcap_connection_error(ese))
self.assertTrue(redi_email.send_email_input_data_unchanged(ese))
def generate_output(person_tree, redcap_settings, email_settings, data_repository):
# redi.configure_logger(system_log_file_full_path)
# the global dictionary to be returned
report_data = {
'errors': []
}
"""
For each person we keep a count for each form type:
subject_details = array(
'person_A' => array('form_1': 1, 'form_2': 10, ...
'person_B' => array('form_1': 1, 'form_2': 10, ...
...
)
"""
subject_details = {}
# For each form type we keep a global count
form_details = {}
# count how many `person` elements are parsed
person_count = 0
root = person_tree.getroot()
persons = root.xpath('//person')
try:
# Communication with redcap
redcapClientObject = redcapClient(
redcap_settings['redcap_uri'],redcap_settings['token'], redcap_settings['verify_ssl'])
except RequestException:
redi_email.send_email_redcap_connection_error(email_settings)
quit()
rate_limiter_value_in_redcap = float(redcap_settings['rate_limiter_value_in_redcap'])
ideal_time_per_request = 60 / float(rate_limiter_value_in_redcap)
time_stamp_after_request = 0
# main loop for each person
for person in persons:
time_begin = datetime.datetime.now()
person_count += 1
study_id = (person.xpath('study_id') or [None])[0]
if study_id is None:
raise Exception('Expected a valid value for study_id')
# count how many csv fragments are created per person
event_count = 0
logger.info('Start sending data for study_id: %s' % study_id.text)
forms = person.xpath('./all_form_events/form')
# loop through the forms of one person
for form in forms:
form_name = form.xpath('name')[0].text
form_key = 'Total_' + form_name + '_Forms'
study_id_key = study_id.text
# init dictionary for a new person in (study_id)
if study_id_key not in subject_details:
subject_details[study_id_key] = {}
if not form_key in subject_details[study_id_key]:
subject_details[study_id_key][form_key] = 0
if form_key not in form_details:
form_details[form_key] = 0
logger.debug(
'parsing study_id ' +
study_id.text +
' form: ' +
form_name)
# loop through the events of one form
for event in form.xpath('event'):
event_status = event.findtext('status')
if event_status == 'sent':
continue
event_count += 1
try:
import_dict = {
redcapClientObject.project.def_field: study_id.text}
import_dict = create_import_data_json(
import_dict,
event)
json_data_dict = import_dict['json_data']
contains_data = import_dict['contains_data']
time_lapse_since_last_request = time.time(
) - time_stamp_after_request
sleepTime = max(
ideal_time_per_request -
time_lapse_since_last_request,
0)
# print 'Sleep for: %s seconds' % sleepTime
time.sleep(sleepTime)
if (0 == event_count % 50):
logger.info('Requests sent: %s' % (event_count))
# to speedup testing uncomment the following line
# if (0 == event_count % 2) : continue
try:
found_error = False
response = redcapClientObject.send_data_to_redcap([json_data_dict], overwrite = True)
status = event.find('status')
if status is not None:
status.text = 'sent'
else:
status_element = etree.Element("status")
status_element.text = 'sent'
event.append(status_element)
data_repository.store(person_tree)
except RedcapError as e:
found_error = handle_errors_in_redcap_xml_response(
e.message,
report_data)
time_stamp_after_request = time.time()
if contains_data:
if not found_error:
# if no errors encountered update event counters
subject_details[study_id_key][form_key] += 1
form_details[form_key] += 1
except Exception as e:
logger.error(e.message)
raise
time_end = datetime.datetime.now()
logger.info("Total execution time for study_id %s was %s" % (study_id_key, (time_end - time_begin)))
logger.info("Total REDCap requests sent: %s \n" % (event_count))
report_data.update({
'total_subjects': person_count,
'form_details': form_details,
'subject_details': subject_details,
'errors': report_data['errors']
})
logger.debug('report_data ' + repr(report_data))
return report_data
