def test_failed(self): """ Verify return false when email is not sent""" ese = self.email_settings self.assertFalse(redi_email.send_email_redcap_connection_error(ese)) self.assertFalse(redi_email.send_email_input_data_unchanged(ese))
def generate_output(person_tree, redcap_settings, email_settings, data_repository): # redi.configure_logger(system_log_file_full_path) # the global dictionary to be returned report_data = {'errors': []} """ For each person we keep a count for each form type: subject_details = array( 'person_A' => array('form_1': 1, 'form_2': 10, ... 'person_B' => array('form_1': 1, 'form_2': 10, ... ... ) """ subject_details = {} # For each form type we keep a global count form_details = {} # count how many `person` elements are parsed person_count = 0 root = person_tree.getroot() persons = root.xpath('//person') try: # Communication with redcap redcapClientObject = redcapClient(redcap_settings['redcap_uri'], redcap_settings['token'], redcap_settings['verify_ssl']) except RequestException: redi_email.send_email_redcap_connection_error(email_settings) quit() rate_limiter_value_in_redcap = float( redcap_settings['rate_limiter_value_in_redcap']) ideal_time_per_request = 60 / float(rate_limiter_value_in_redcap) time_stamp_after_request = 0 # main loop for each person for person in persons: time_begin = datetime.datetime.now() person_count += 1 study_id = (person.xpath('study_id') or [None])[0] if study_id is None: raise Exception('Expected a valid value for study_id') # count how many csv fragments are created per person event_count = 0 logger.info('Start sending data for study_id: %s' % study_id.text) forms = person.xpath('./all_form_events/form') # loop through the forms of one person for form in forms: form_name = form.xpath('name')[0].text form_key = 'Total_' + form_name + '_Forms' study_id_key = study_id.text # init dictionary for a new person in (study_id) if study_id_key not in subject_details: subject_details[study_id_key] = {} if not form_key in subject_details[study_id_key]: subject_details[study_id_key][form_key] = 0 if form_key not in form_details: form_details[form_key] = 0 logger.debug('parsing study_id ' + study_id.text + ' form: ' + form_name) # loop through the events of one form for event in form.xpath('event'): event_status = event.findtext('status') if event_status == 'sent': continue event_count += 1 try: import_dict = { redcapClientObject.project.def_field: study_id.text } import_dict = create_import_data_json(import_dict, event) json_data_dict = import_dict['json_data'] contains_data = import_dict['contains_data'] time_lapse_since_last_request = time.time( ) - time_stamp_after_request sleepTime = max( ideal_time_per_request - time_lapse_since_last_request, 0) # print 'Sleep for: %s seconds' % sleepTime time.sleep(sleepTime) if (0 == event_count % 50): logger.info('Requests sent: %s' % (event_count)) # to speedup testing uncomment the following line # if (0 == event_count % 2) : continue try: found_error = False response = redcapClientObject.send_data_to_redcap( [json_data_dict], overwrite=True) status = event.find('status') if status is not None: status.text = 'sent' else: status_element = etree.Element("status") status_element.text = 'sent' event.append(status_element) data_repository.store(person_tree) except RedcapError as e: found_error = handle_errors_in_redcap_xml_response( e.message, report_data) time_stamp_after_request = time.time() if contains_data: if not found_error: # if no errors encountered update event counters subject_details[study_id_key][form_key] += 1 form_details[form_key] += 1 except Exception as e: logger.error(e.message) raise time_end = datetime.datetime.now() logger.info("Total execution time for study_id %s was %s" % (study_id_key, (time_end - time_begin))) logger.info("Total REDCap requests sent: %s \n" % (event_count)) report_data.update({ 'total_subjects': person_count, 'form_details': form_details, 'subject_details': subject_details, 'errors': report_data['errors'] }) logger.debug('report_data ' + repr(report_data)) return report_data
def test_success(self): """ Verify return true when email is sent""" ese = self.email_settings self.assertTrue(redi_email.send_email_redcap_connection_error(ese)) self.assertTrue(redi_email.send_email_input_data_unchanged(ese))
def generate_output(person_tree, redcap_settings, email_settings, data_repository): # redi.configure_logger(system_log_file_full_path) # the global dictionary to be returned report_data = { 'errors': [] } """ For each person we keep a count for each form type: subject_details = array( 'person_A' => array('form_1': 1, 'form_2': 10, ... 'person_B' => array('form_1': 1, 'form_2': 10, ... ... ) """ subject_details = {} # For each form type we keep a global count form_details = {} # count how many `person` elements are parsed person_count = 0 root = person_tree.getroot() persons = root.xpath('//person') try: # Communication with redcap redcapClientObject = redcapClient( redcap_settings['redcap_uri'],redcap_settings['token'], redcap_settings['verify_ssl']) except RequestException: redi_email.send_email_redcap_connection_error(email_settings) quit() rate_limiter_value_in_redcap = float(redcap_settings['rate_limiter_value_in_redcap']) ideal_time_per_request = 60 / float(rate_limiter_value_in_redcap) time_stamp_after_request = 0 # main loop for each person for person in persons: time_begin = datetime.datetime.now() person_count += 1 study_id = (person.xpath('study_id') or [None])[0] if study_id is None: raise Exception('Expected a valid value for study_id') # count how many csv fragments are created per person event_count = 0 logger.info('Start sending data for study_id: %s' % study_id.text) forms = person.xpath('./all_form_events/form') # loop through the forms of one person for form in forms: form_name = form.xpath('name')[0].text form_key = 'Total_' + form_name + '_Forms' study_id_key = study_id.text # init dictionary for a new person in (study_id) if study_id_key not in subject_details: subject_details[study_id_key] = {} if not form_key in subject_details[study_id_key]: subject_details[study_id_key][form_key] = 0 if form_key not in form_details: form_details[form_key] = 0 logger.debug( 'parsing study_id ' + study_id.text + ' form: ' + form_name) # loop through the events of one form for event in form.xpath('event'): event_status = event.findtext('status') if event_status == 'sent': continue event_count += 1 try: import_dict = { redcapClientObject.project.def_field: study_id.text} import_dict = create_import_data_json( import_dict, event) json_data_dict = import_dict['json_data'] contains_data = import_dict['contains_data'] time_lapse_since_last_request = time.time( ) - time_stamp_after_request sleepTime = max( ideal_time_per_request - time_lapse_since_last_request, 0) # print 'Sleep for: %s seconds' % sleepTime time.sleep(sleepTime) if (0 == event_count % 50): logger.info('Requests sent: %s' % (event_count)) # to speedup testing uncomment the following line # if (0 == event_count % 2) : continue try: found_error = False response = redcapClientObject.send_data_to_redcap([json_data_dict], overwrite = True) status = event.find('status') if status is not None: status.text = 'sent' else: status_element = etree.Element("status") status_element.text = 'sent' event.append(status_element) data_repository.store(person_tree) except RedcapError as e: found_error = handle_errors_in_redcap_xml_response( e.message, report_data) time_stamp_after_request = time.time() if contains_data: if not found_error: # if no errors encountered update event counters subject_details[study_id_key][form_key] += 1 form_details[form_key] += 1 except Exception as e: logger.error(e.message) raise time_end = datetime.datetime.now() logger.info("Total execution time for study_id %s was %s" % (study_id_key, (time_end - time_begin))) logger.info("Total REDCap requests sent: %s \n" % (event_count)) report_data.update({ 'total_subjects': person_count, 'form_details': form_details, 'subject_details': subject_details, 'errors': report_data['errors'] }) logger.debug('report_data ' + repr(report_data)) return report_data