def getExternalRecords(self, pds, subject, lbls): er_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD) try: pds_records = er_rh.get(external_system_url=pds.data_source.url, path=pds.path, subject_id=subject['id']) time.sleep(0.05) except PageNotFound: pds_records = [] r = [] for ex_rec in pds_records: # Convert ehb-client object to JSON and then parse as py dict e = json.loads(ex_rec.json_from_identity(ex_rec)) # Map label descriptions from the eHB to External Records for label in lbls: if e['label'] == label['id']: if label['label'] == '': e['label_desc'] = 'Record' else: e['label_desc'] = label['label'] e['pds'] = pds.id r.append(e) return r
def handle(self, *args, **options): def subject_threading(self, pds, s_id, lbls, user, cache_key): # get all records associated with subject records = self.get_subject_records(pds, s_id, lbls) for record in records: r_id = record['id'] r_name = record['record_id'] self.cache_redcap_form_complete(pds, user, cache_key, s_id, r_id, r_name) return def clear_cache(cache_key): cache.delete(cache_key) start = time.time() er_label_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD_LABEL) lbls = er_label_rh.query() protocols = self.get_protocols(options['protocol_id']) for protocol in protocols: protocoldatasources = self.get_protocoldatasource( protocol) # get redcap protocol datasource for pds in protocoldatasources: print("\n Caching protocol datasource {} \n".format(pds)) user = self.get_protocol_user( protocol) # get eig user in protocol subject_id_list = self.get_protocol_subjects( protocol, lbls) # get all subjects in protocol try: cache_key = 'protocoldatasource{0}'.format( pds.id) + '_redcap_completion_codes' cache.delete(cache_key) except AttributeError: # protocoldatasource wasn't properly configured print(str(pds) + ' was skipped') continue threads = [ ] # array to hold all threads, length is # of records per subj for s_id in subject_id_list: # for every subject # creating threads curr_thread = threading.Thread(target=subject_threading, args=( self, pds, s_id, lbls, user, cache_key, )) threads.append(curr_thread) curr_thread.start() # make sure every thread has finished for t in threads: t.join() elapsed = time.time() - start print("total caching time" + str(elapsed)) print("caching records table complete")
def handle(self, *args, **options): erl_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD_LABEL) labels = erl_rh.query() cache.set('ehb_labels', labels) if hasattr(cache, 'persist'): cache.persist('ehb_labels') print("caching of ehb_labels complete")
def getExternalRecords(self, pds, subject, lbls): """Get external records for a subject/protocol combination. Returns external records for a given subject from a given protocol after labelling them with EHB labels and the protocol id. """ # Get the appropriate request handler for external records. er_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD) try: # Retrieve the external records for the subject/protocol. # Path argument is needed for ehb_datasources to make request. pds_records = er_rh.get(external_system_url=pds.data_source.url, path=pds.path, subject_id=subject['id']) # TODO: why is this necessary? time.sleep(0.05) except PageNotFound: # Return an empty array if the subject/protocol is not found. pds_records = [] # Result array. r = [] for ex_rec in pds_records: # Convert ehb-client object to JSON and then parse as py dict. # TODO: the external record object should have this capability. # We should examine this to refactor for simplicity. It is possible # to remove json.loads for each json_from_identity call. e = json.loads(ex_rec.json_from_identity(ex_rec)) # Map label descriptions from the eHB to External Records. for label in lbls: if e['label'] == label['id']: if label['label'] == '': e['label_desc'] = 'Record' else: e['label_desc'] = label['label'] # Add the protocol datasource id to the external record. e['pds'] = pds.id r.append(e) return r
class BRPApiView(APIView): # Cache cache = _cache # Honest Broker Request Handelers s_rh = ServiceClient.get_rh_for(record_type=ServiceClient.SUBJECT) o_rh = ServiceClient.get_rh_for(record_type=ServiceClient.ORGANIZATION) es_rh = ServiceClient.get_rh_for(record_type=ServiceClient.EXTERNAL_SYSTEM) er_rh = ServiceClient.get_rh_for(record_type=ServiceClient.EXTERNAL_RECORD) erl_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD_LABEL) # noqa err_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD_RELATION) # noqa g_rh = ServiceClient.get_rh_for(record_type=ServiceClient.GROUP) relationship_HB_handler = ServiceClient.get_rh_for( record_type=ServiceClient.RELATIONSHIP) relationship_type_HB_handler = ServiceClient.get_rh_for( record_type=ServiceClient.RELATIONSHIP_TYPE) # Utility Classes subject_utils = SubjectUtils record_utils = RecordUtils
def getExternalRecords(self, pds, subject, lbls): er_rh = ServiceClient.get_rh_for(record_type=ServiceClient.EXTERNAL_RECORD) try: pds_records = er_rh.get( external_system_url=pds.data_source.url, path=pds.path, subject_id=subject['id']) time.sleep(0.05) except PageNotFound: pds_records = [] r = [] for ex_rec in pds_records: # Convert ehb-client object to JSON and then parse as py dict e = json.loads(ex_rec.json_from_identity(ex_rec)) # Map label descriptions from the eHB to External Records for label in lbls: if e['label'] == label['id']: if label['label'] == '': e['label_desc'] = 'Record' else: e['label_desc'] = label['label'] e['pds'] = pds.id r.append(e) return r
def handle(self, *args, **options): if options['protocol']: if options['continue']: protocols = Protocol.objects.filter( id__gte=options['protocol']).order_by('id').all() else: try: protocols = Protocol.objects.filter( id=options['protocol']).all() except Protocol.DoesNotExist: print('Protocol specified does not exist') return else: protocols = Protocol.objects.order_by('id').all() g_rh = ServiceClient.get_rh_for(record_type=ServiceClient.GROUP) errors = [] msg = None for protocol in protocols: print('Checking {0} (ID:{1}) Group: {2}'.format( protocol.name, protocol.id, protocol.ehb_group_name())) try: g_rh.get(name=protocol.ehb_group_name()) except RequestedRangeNotSatisfiable: msg = ('Unable to find expected group: {0}'.format( protocol.ehb_group_name())) errors.append(msg) print(msg) msg = None # Check for pds groups subjects = protocol.getSubjects() if subjects: for subject in subjects: try: SubjectUtils.get_protocol_subject_record_group( protocol, subject) except RequestedRangeNotSatisfiable as e: msg = ( 'Warning: Group not found under Protocol: {0} Group name expected: {1}' .format(protocol.name, e.errmsg)) except Organization.DoesNotExist: msg = ( 'Warning: Could not find associated Organization for protocol: {0}, eHB Org ID: {1} Likely eHB/BRP organizaiton mismatch' .format(protocol.name, subject.organization_id)) except BadStatusLine: msg = ( 'Warning: Malformed Group Detected for Protocol: {0} Subject ID: {1} Likely Bad Organization ID' .format(protocol.name, subject.id)) if msg: errors.append(msg) print(msg) msg = None if csv: with open('group_report_{0}.csv'.format( datetime.now().strftime('%Y%m%d.%H:%M.%S')), 'w', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) for error in errors: csv_writer.writerow([error])
def cache_records(self, protocol_id): protocol_id = protocol_id[0] if protocol_id == 'all': protocols = Protocol.objects.all() else: protocols = Protocol.objects.filter(id=int(protocol_id)).all() er_label_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD_LABEL) lbls = er_label_rh.query() print('Caching {0} protocol(s)...'.format(len(protocols))) for protocol in protocols: print('Caching {}'.format(protocol)) subjects = protocol.getSubjects() organizations = protocol.organizations.all() if subjects: subs = [eHBSubjectSerializer(sub).data for sub in subjects] else: continue ehb_orgs = [] # We can't rely on Ids being consistent across apps so we must # append the name here for display downstream. for o in organizations: ehb_orgs.append(o.getEhbServiceInstance()) # Check if the protocol has external IDs configured. If so retrieve them manageExternalIDs = False protocoldatasources = protocol.getProtocolDataSources() for pds in protocoldatasources: if pds.driver == 3: ExIdSource = pds manageExternalIDs = True if manageExternalIDs: try: config = json.loads(ExIdSource.driver_configuration) if 'sort_on' in list(config.keys()): # er_label_rh = ServiceClient.get_rh_for(record_type=ServiceClient.EXTERNAL_RECORD_LABEL) # lbl = er_label_rh.get(id=config['sort_on']) lbl = '' addl_id_column = lbl except: raise pass for sub in subs: sub['external_records'] = [] sub['external_ids'] = [] sub['organization'] = sub['organization_id'] sub.pop('organization_id') for pds in protocoldatasources: try: sub['external_records'].extend( self.getExternalRecords(pds, sub, lbls)) except: print("there was an error processing external records") print("subject DB id:") print(sub['id']) print("protocol data source:") print(pds) pass if manageExternalIDs: # Break out external ids into a separate object for ease of use for record in sub['external_records']: if record['external_system'] == 3: try: sub['external_ids'].append(record) except: print( "an error occured getting external records" ) print(sub['external_ids']) for ehb_org in ehb_orgs: if sub['organization'] == ehb_org.id: sub['organization_name'] = ehb_org.name cache_key = 'protocol{0}_sub_data'.format(protocol.id) cache.set(cache_key, json.dumps(subs)) cache.persist(cache_key)
def post(self, request, **kwargs): context = self.get_context_data(**kwargs) def rec_id_validator(new_record_id, include_path): return SubjectUtils.validate_new_record_id(context['pds'], context['subject'], new_record_id, include_path) try: grp = SubjectUtils.get_protocol_subject_record_group( self.pds.protocol, self.subject) grp.client_key = self.pds.protocol._settings_prop( 'CLIENT_KEY', 'key', '') rec_id_prefix = '' label_id = request.POST.get('label_id', 1) if grp: rec_id_prefix = grp.ehb_key else: request.META[ 'action'] = 'Subject record group not found for {0}'.format( context['subject'].id) request.META['error'] = True raise Exception('No subject record group found') # Try to process the new record form try: rec_id = self.driver.process_new_record_form( request=request, record_id_prefix=rec_id_prefix, record_id_validator=rec_id_validator) # If we have successfully created the record. Make sure it is in the eHB. try: self.record_id = SubjectUtils.create_new_ehb_external_record( self.pds, request.user, self.subject, rec_id, label_id).id if self.check_cache(): self.update_cache() return HttpResponseRedirect(self.start_path) except RecordCreationError as rce: # exception from the eHB request.META['action'] = rce.errmsg request.META['error'] = True context['errors'].append( ('The record could not be created on the ' 'electronic Honest Broker. Please contact a ' 'system administrator. There could be a ' 'connection problem.')) except IgnoreEhbExceptions as iee: # TODO this is a hack until BRP can create Nautilus records rec_id = iee.record_id ehb_rec_id = '' try: # this will create the ehb external_record entry # and add that record to the subject's record group self.record_id = SubjectUtils.create_new_ehb_external_record( self.pds, request.user, self.subject, rec_id, label_id).id if self.check_cache(): self.update_cache() self.start_path = '{0}/dataentry/protocoldatasource/{1}/subject/{2}/record/{3}/start/'.format( self.service_client.self_root_path, self.pds.id, self.subject.id, self.record_id) return HttpResponseRedirect(self.start_path) except RecordCreationError as rce: # exception from the eHB request.META['action'] = rce.errmsg request.META['error'] = True record_already_exists = 6 cause = rce.raw_cause if cause and len( cause) == 1 and record_already_exists == cause[0]: er_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD) ehb_recs = er_rh.get( external_system_url=self.pds.data_source.url, subject_id=self.subject.id, path=self.pds.path) ehb_rec_id = None for record in ehb_recs: if record.record_id == rec_id: ehb_rec_id = record.id if ehb_rec_id: self.start_path = '%s/dataentry/protocoldatasource/%s/subject/%s/record/%s/start' % ( ServiceClient.self_root_path, self.pds.id, self.subject.id, ehb_rec_id) else: context['errors'].append( 'This ID has already been assigned to another subject.' ) return render_to_response( 'pds_dataentry_rec_create.html', context, context_instance=RequestContext(request)) else: request.META['action'] = 'Record could not be created' request.META['error'] = True context['errors'].append( ('The record could not be created on the ' 'electronic Honest Broker. Please contact ' 'a system administrator. There could be a' ' connection problem.')) except: return HttpResponse('Unknown Error') return HttpResponseRedirect(self.start_path) except RecordCreationError as rce: # Handle errors in the form. request.META['action'] = rce.errmsg request.META['error'] = True context['errors'].append(rce.cause) return render_to_response('pds_dataentry_rec_create.html', context, context_instance=RequestContext(request))
def post(self, request, **kwargs): context = self.get_context_data(**kwargs) def rec_id_validator(new_record_id, include_path): return SubjectUtils.validate_new_record_id( context['pds'], context['subject'], new_record_id, include_path) try: grp = SubjectUtils.get_protocol_subject_record_group( self.pds.protocol, self.subject) grp.client_key = self.pds.protocol._settings_prop( 'CLIENT_KEY', 'key', '') rec_id_prefix = '' label_id = request.POST.get('label_id', 1) if grp: rec_id_prefix = grp.ehb_key else: request.META['action'] = 'Subject record group not found for {0}'.format(context['subject'].id) request.META['error'] = True raise Exception('No subject record group found') # Try to process the new record form try: rec_id = self.driver.process_new_record_form( request=request, record_id_prefix=rec_id_prefix, record_id_validator=rec_id_validator ) # If we have successfully created the record. Make sure it is in the eHB. try: self.record_id = SubjectUtils.create_new_ehb_external_record( self.pds, request.user, self.subject, rec_id, label_id).id if self.check_cache(): self.update_cache() return HttpResponseRedirect(self.start_path) except RecordCreationError as rce: # exception from the eHB request.META['action'] = rce.errmsg request.META['error'] = True context['errors'].append(( 'The record could not be created on the ' 'electronic Honest Broker. Please contact a ' 'system administrator. There could be a ' 'connection problem.')) except IgnoreEhbExceptions as iee: # TODO this is a hack until BRP can create Nautilus records rec_id = iee.record_id ehb_rec_id = '' try: # this will create the ehb external_record entry # and add that record to the subject's record group self.record_id = SubjectUtils.create_new_ehb_external_record( self.pds, request.user, self.subject, rec_id, label_id).id if self.check_cache(): self.update_cache() self.start_path = '{0}/dataentry/protocoldatasource/{1}/subject/{2}/record/{3}/start/'.format( self.service_client.self_root_path, self.pds.id, self.subject.id, self.record_id) return HttpResponseRedirect(self.start_path) except RecordCreationError as rce: # exception from the eHB request.META['action'] = rce.errmsg request.META['error'] = True record_already_exists = 6 cause = rce.raw_cause if cause and len(cause) == 1 and record_already_exists == cause[0]: er_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD) ehb_recs = er_rh.get( external_system_url=self.pds.data_source.url, subject_id=self.subject.id, path=self.pds.path) ehb_rec_id = None for record in ehb_recs: if record.record_id == rec_id: ehb_rec_id = record.id if ehb_rec_id: self.start_path = '%s/dataentry/protocoldatasource/%s/subject/%s/record/%s/start' % ( ServiceClient.self_root_path, self.pds.id, self.subject.id, ehb_rec_id) else: context['errors'].append( 'This ID has already been assigned to another subject.') return render_to_response( 'pds_dataentry_rec_create.html', context, context_instance=RequestContext(request) ) else: request.META['action'] = 'Record could not be created' request.META['error'] = True context['errors'].append(( 'The record could not be created on the ' 'electronic Honest Broker. Please contact ' 'a system administrator. There could be a' ' connection problem.')) except: return HttpResponse('Unknown Error') return HttpResponseRedirect(self.start_path) except RecordCreationError as rce: # Handle errors in the form. request.META['action'] = rce.errmsg request.META['error'] = True context['errors'].append(rce.cause) return render_to_response( 'pds_dataentry_rec_create.html', context, context_instance=RequestContext(request) )
def cache_records(self, protocol_id): """Cache subject records from a given protocol locally.""" # TODO: consider passing in a list if we ever need to cache a small # number of protocols at one time. look at why only using first item in # list. protocol_id = protocol_id[0] if protocol_id == 'all': # Special "all" protocol gets all protocols. protocols = Protocol.objects.all() else: protocols = Protocol.objects.filter(id=int(protocol_id)).all() # Get external record label request handler. er_label_rh = ServiceClient.get_rh_for( record_type=ServiceClient.EXTERNAL_RECORD_LABEL) # Retrieve the actual external record labels. lbls = er_label_rh.query() # Tell user how many protocols are being cached. print('Caching {0} protocol(s)...'.format(len(protocols))) for protocol in protocols: # Tell user which protocol is being cached. print('Caching {}'.format(protocol)) # Get list of subjects and organizations in the protocol. subjects = protocol.getSubjects() organizations = protocol.organizations.all() # Serialize retrieved subjects or continue if there are none. if subjects: subs = [eHBSubjectSerializer(sub).data for sub in subjects] else: continue ehb_orgs = [] # We can't rely on Ids being consistent across apps so we must # append the name here for display downstream. for o in organizations: ehb_orgs.append(o.getEhbServiceInstance()) # TODO: Explain this block, down to the `for sub in subs` loop. # Check if the protocol has external IDs configured. # If so, retrieve them. manageExternalIDs = False protocoldatasources = protocol.getProtocolDataSources() for pds in protocoldatasources: if pds.driver == 3: ExIdSource = pds manageExternalIDs = True if manageExternalIDs: try: config = json.loads(ExIdSource.driver_configuration) if 'sort_on' in list(config.keys()): # er_label_rh = ServiceClient.get_rh_for( # record_type=ServiceClient.EXTERNAL_RECORD_LABEL) # lbl = er_label_rh.get(id=config['sort_on']) lbl = '' addl_id_column = lbl # noqa except: raise pass # Transform subjects for ease of use. for sub in subs: # Initialize new fields. sub['external_records'] = [] sub['external_ids'] = [] sub['organization'] = sub['organization_id'] sub.pop('organization_id') # Add external records from all data sources. for pds in protocoldatasources: try: sub['external_records'].extend( self.getExternalRecords(pds, sub, lbls)) except: print("there was an error processing external records") print("subject DB id:") print(sub['id']) print("protocol data source:") print(pds) pass # TODO: Explain this block. if manageExternalIDs: # Break out external ids into a separate object for ease of # use. for record in sub['external_records']: if record['external_system'] == 3: try: sub['external_ids'].append(record) except: print( "an error occured getting external records" ) print(sub['external_ids']) # Add organization name to subject record for display, since # organization IDs can vary across apps. (?) for ehb_org in ehb_orgs: if sub['organization'] == ehb_org.id: sub['organization_name'] = ehb_org.name # Cache the array of subjects. cache_key = 'protocol{0}_sub_data'.format(protocol.id) cache.set(cache_key, json.dumps(subs)) cache.persist(cache_key)
def handle(self, *args, **options): if options['protocol']: if options['continue']: protocols = Protocol.objects.filter(id__gte=options['protocol']).order_by('id').all() else: try: protocols = Protocol.objects.filter(id=options['protocol']).all() except Protocol.DoesNotExist: print('Protocol specified does not exist') return else: protocols = Protocol.objects.order_by('id').all() g_rh = ServiceClient.get_rh_for(record_type=ServiceClient.GROUP) errors = [] msg = None for protocol in protocols: print('Checking {0} (ID:{1}) Group: {2}'.format( protocol.name, protocol.id, protocol.ehb_group_name())) try: g_rh.get(name=protocol.ehb_group_name()) except RequestedRangeNotSatisfiable: msg = ('Unable to find expected group: {0}'.format(protocol.ehb_group_name())) errors.append(msg) print(msg) msg = None # Check for pds groups subjects = protocol.getSubjects() if subjects: for subject in subjects: try: SubjectUtils.get_protocol_subject_record_group(protocol, subject) except RequestedRangeNotSatisfiable as e: msg = ('Warning: Group not found under Protocol: {0} Group name expected: {1}'.format( protocol.name, e.errmsg )) except Organization.DoesNotExist: msg = ('Warning: Could not find associated Organization for protocol: {0}, eHB Org ID: {1} Likely eHB/BRP organizaiton mismatch'.format( protocol.name, subject.organization_id )) except BadStatusLine: msg = ('Warning: Malformed Group Detected for Protocol: {0} Subject ID: {1} Likely Bad Organization ID'.format( protocol.name, subject.id )) if msg: errors.append(msg) print(msg) msg = None if csv: with open('group_report_{0}.csv'.format(datetime.now().strftime('%Y%m%d.%H:%M.%S')), 'w', newline='') as csvfile: csv_writer = csv.writer(csvfile, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) for error in errors: csv_writer.writerow([error])
def cache_records(self, protocol_id): protocol_id = protocol_id[0] if protocol_id == 'all': protocols = Protocol.objects.all() else: protocols = Protocol.objects.filter(id=int(protocol_id)).all() er_label_rh = ServiceClient.get_rh_for(record_type=ServiceClient.EXTERNAL_RECORD_LABEL) lbls = er_label_rh.query() print('Caching {0} protocol(s)...'.format(len(protocols))) for protocol in protocols: print('Caching {}'.format(protocol)) subjects = protocol.getSubjects() organizations = protocol.organizations.all() if subjects: subs = [eHBSubjectSerializer(sub).data for sub in subjects] else: continue ehb_orgs = [] # We can't rely on Ids being consistent across apps so we must # append the name here for display downstream. for o in organizations: ehb_orgs.append(o.getEhbServiceInstance()) # Check if the protocol has external IDs configured. If so retrieve them manageExternalIDs = False protocoldatasources = protocol.getProtocolDataSources() for pds in protocoldatasources: if pds.driver == 3: ExIdSource = pds manageExternalIDs = True if manageExternalIDs: try: config = json.loads(ExIdSource.driver_configuration) if 'sort_on' in list(config.keys()): # er_label_rh = ServiceClient.get_rh_for(record_type=ServiceClient.EXTERNAL_RECORD_LABEL) # lbl = er_label_rh.get(id=config['sort_on']) lbl = '' addl_id_column = lbl except: raise pass for sub in subs: sub['external_records'] = [] sub['external_ids'] = [] sub['organization'] = sub['organization_id'] sub.pop('organization_id') for pds in protocoldatasources: sub['external_records'].extend(self.getExternalRecords(pds, sub, lbls)) if manageExternalIDs: # Break out external ids into a separate object for ease of use for record in sub['external_records']: if record['external_system'] == 3: sub['external_ids'].append(record) for ehb_org in ehb_orgs: if sub['organization'] == ehb_org.id: sub['organization_name'] = ehb_org.name cache_key = 'protocol{0}_sub_data'.format(protocol.id) cache.set(cache_key, json.dumps(subs)) cache.persist(cache_key)