class BaseImporter: logger = DaisyLogger(__name__) def process_contacts(self, project_dict): local_custodians = [] local_personnel = [] external_contacts = [] home_organisation = Partner.objects.get(acronym=settings.COMPANY) for contact_dict in project_dict.get('contacts', []): first_name = contact_dict.get('first_name').strip() last_name = contact_dict.get('last_name').strip() email = contact_dict.get('email','').strip() full_name = "{} {}".format(first_name, last_name) role_name = contact_dict.get('role') if home_organisation.elu_accession == contact_dict.get('institution').strip(): user = (User.objects.filter(first_name__icontains=first_name.lower(), last_name__icontains=last_name.lower()) | User.objects.filter( first_name__icontains=first_name.upper(), last_name__icontains=last_name.upper())).first() if user is None: self.logger.warning('no user found for %s an inactive user will be created', full_name) usr_name = first_name.lower() + '.' + last_name.lower() user = User.objects.create(username=usr_name, password='', first_name=first_name, last_name=last_name, is_active=False, email=email, ) user.staff = True if role_name == PRINCIPAL_INVESTIGATOR: g = Group.objects.get(name=GroupConstants.VIP.value) user.groups.add(g) user.save() if role_name == PRINCIPAL_INVESTIGATOR: local_custodians.append(user) else: local_personnel.append(user) else: contact = (Contact.objects.filter(first_name__icontains=first_name.lower(), last_name__icontains=last_name.lower()) | Contact.objects.filter( first_name__icontains=first_name.upper(), last_name__icontains=last_name.upper())).first() if contact is None: contact_type_pi, _ = ContactType.objects.get_or_create(name=role_name) contact, _ = Contact.objects.get_or_create( first_name=first_name, last_name=last_name, email=email, type=contact_type_pi ) affiliation = Partner.objects.get(elu_accession=contact_dict.get('institution')) if affiliation: contact.partners.add(affiliation) contact.save() external_contacts.append(contact) return local_custodians, local_personnel, external_contacts
from django.urls import reverse_lazy from django.views.generic import DetailView, UpdateView from haystack.query import SearchQuerySet from django.db import IntegrityError, transaction from core.constants import Permissions from core.forms import DataDeclarationForm, DataDeclarationSubFormOther, DataDeclarationSubFormNew, \ DataDeclarationSubFormFromExisting, DataDeclarationEditForm from core.forms.data_declaration import RestrictionFormset from core.models import Dataset, Partner, DataDeclaration, UseRestriction from core.utils import DaisyLogger from core.permissions import permission_required, CheckerMixin, constants from django.core.paginator import Paginator from django.http import JsonResponse, HttpResponseBadRequest from django.db.models import Q log = DaisyLogger(__name__) DATA_DECLARATIONS_SUB_FORMS = [ (DataDeclarationSubFormFromExisting, 'data_declaration_sub_form_existing.html'), (DataDeclarationSubFormNew, 'data_declaration_sub_form_new.html'), (DataDeclarationSubFormOther, 'data_declaration_sub_form_other.html') ] def data_declarations_add_sub_form(request): declaration_type = request.GET['declaration_type'] dataset_id = int(request.GET['dataset_id']) dataset = get_object_or_404(Dataset, id=dataset_id) declaration_type = int(declaration_type) form_class, template = DATA_DECLARATIONS_SUB_FORMS[declaration_type]
class BaseImporter: """ Abstract base class for an importer. Provides common functions for opening/parsing/validating JSON files. Take a look on `ProjectsImporter` or `DatasetsImporter` for information how an implementation should look like. """ class DateImportException(Exception): pass logger = DaisyLogger(__name__) @property def json_schema_validator(self): """ This validator will be used against the imported data """ raise NotImplementedError('You must implement `json_schema_validator` in your importer class') @property def json_schema_uri(self): """ This attribute is used for detecting whether the importer can handle given json """ raise NotImplementedError('You must implement `json_schema_uri` in your importer class') def can_process_json(self, json_string: str) -> bool: """ Checks whether the imported JSON has the same "$schema" URI as the importer class (in `json_schema_uri` property) """ try: object = json.loads(json_string) return self.can_process_object(object) except: message = f'Couldn\'t check if the imported object has same "$schema" as the importer ({self.__class__.__name__}: {self.json_schema_uri}) - something went wrong while parsing the file' self.logger.warn(message) return False def can_process_object(self, json_object: Dict) -> bool: """ Checks whether the object has the same "$schema" URI as the importer class (in `json_schema_uri` property) """ if not json_object.get('$schema', False): self.logger.debug('The imported object has no "$schema" attribute') return False if self.json_schema_uri == json_object.get('$schema'): message = f'The imported object has the same "$schema" ({self.json_schema_uri}) as the importer ({self.__class__.__name__})' self.logger.debug(message) return True schema_name = json_object.get('$schema') message = f'The imported object has different "$schema" ({schema_name}) than the importer ({self.__class__.__name__}: {self.json_schema_uri})' self.logger.debug(message) return False def import_json_file(self, path_to_the_file: str, stop_on_error=False, verbose=False, validate=True) -> bool: """ Opens, loads and imports a JSON file. """ self.logger.info(f'Opening the file: {path_to_the_file}') with open(path_to_the_file, encoding='utf-8') as json_file: json_file_contents = json_file.read() result = self.import_json(json_file_contents, stop_on_error, verbose) self.logger.info(f'Successfully completed import for the file: {path_to_the_file}') return result def import_json(self, json_string: str, stop_on_error=False, verbose=False, validate=True) -> bool: result = True importer_class_name = self.__class__.__name__ self.logger.info(f'Attempting to use "{importer_class_name}" to parse and import the JSON') json_list = json.loads(json_string)['items'] result = self.import_object_list(json_list, stop_on_error, verbose) status = 'success' if result else 'failed' self.logger.info(f'Import ({importer_class_name}) result: {status}') return result def import_object_list(self, json_list: List[Dict], stop_on_error=False, verbose=False, validate=True) -> bool: """ Validates and imports a list of objects. """ result = True if validate: validator_name = self.json_schema_validator.__class__.__name__ self.logger.debug(f'Validating the file with "{validator_name}" against JSON schema...') self.json_schema_validator.validate_items(json_list, self.logger) self.logger.debug('...JSON schema is OK!') else: self.logger.debug(f'Proceeding without using the validation') count = len(json_list) verb = 'are' if count > 1 else 'is' self.logger.debug(f'There {verb} {count} object(s) to be imported. Starting the process...') for item in json_list: result = self.import_object(item, stop_on_error, verbose) and result self.logger.debug('Finished importing the object(s)') return result def import_object(self, item: Dict, stop_on_error=False, verbose=False): """ Tries to import a single object """ item_name = item.get('name', 'N/A').encode('utf-8') self.logger.debug(f'Trying to import item: "{item_name}"') try: result = self.process_json(item) except Exception as e: self.logger.error('Import failed: ') self.logger.error(str(e)) if verbose: import traceback ex = traceback.format_exception(*sys.exc_info()) self.logger.error('\n'.join([e for e in ex])) if stop_on_error: raise e result = False self.logger.debug(f'Successfully imported item: {item_name}') return result def process_json(self, import_dict): raise NotImplementedError("Abstract method: Implement this method in the child class.") def process_contacts(self, contacts_list: List[Dict]): if not isinstance(contacts_list, list): self.logger.warn('Contact list is not a list... Please check the imported file.') return [], [], [] local_custodians = [] local_personnel = [] external_contacts = [] for contact_dict in contacts_list: first_name = contact_dict.get('first_name').strip() last_name = contact_dict.get('last_name').strip() email = contact_dict.get('email', '').strip() role_name = self.validate_contact_type(contact_dict.get('role')) affiliations = contact_dict.get('affiliations', []) if self.is_local_contact(contact_dict): user = self.process_local_contact(first_name, last_name, email, role_name, affiliations) if role_name == PRINCIPAL_INVESTIGATOR: local_custodians.append(user) else: local_personnel.append(user) else: contact = self.process_external_contact(first_name, last_name, email, role_name, affiliations) external_contacts.append(contact) return local_custodians, local_personnel, external_contacts @staticmethod def process_partner(partner_name): partner, _ = Partner.objects.get_or_create(name=partner_name) return partner def process_date(self, date_string): regex = r'([0-9]{4})-([0-9]{2})-([0-9]{2})' match = re.match(regex, date_string, re.M | re.I) if match: year = match.group(1) month = match.group(2) day = match.group(3) date_str = f"{year}-{month}-{day}" try: r = datetime.strptime(date_str, "%Y-%m-%d").date() return r except (TypeError, ValueError): raise self.DateImportException(f"Couldn't parse the following date: {str(date_string)}") else: raise self.DateImportException(f"Couldn't parse the following date: {str(date_string)}") @staticmethod def is_local_contact(contact_dict): home_organisation = Partner.objects.get(acronym=settings.COMPANY) _is_local_contact = home_organisation.name in contact_dict.get("affiliations") or home_organisation.acronym in contact_dict.get("affiliations") return _is_local_contact def validate_contact_type(self, contact_type): try: contact_type_obj = ContactType.objects.get(name=contact_type) except ContactType.DoesNotExist: self.logger.warning(f'Unknown contact type: {contact_type}. Setting to "Other".') contact_type = 'Other' return contact_type def process_local_contact(self, first_name, last_name, email, role_name, affiliations): user = User.objects.filter(first_name__icontains=first_name,last_name__icontains=last_name) if len(user) > 1: users = User.objects.filter(first_name__icontains=first_name, last_name__icontains=last_name, email=email) if len(users) != 1: msg = 'Something went wrong - there are two contacts with the same first and last name, and it''s impossible to differentiate them' self.logger.warning(msg) user = users.first() elif len(user) == 1: user = user.first() else: user = None if user is None: self.logger.warning(f"No user found for '{first_name} {last_name}' - hence an inactive user will be created") usr_name = first_name.lower() + '.' + last_name.lower() user = User.objects.create(username=usr_name, password='', first_name=first_name, last_name=last_name, is_active=False, email=email) user.staff = True if role_name == PRINCIPAL_INVESTIGATOR: g = Group.objects.get(name=GroupConstants.VIP.value) user.groups.add(g) user.save() return user def process_external_contact(self, first_name, last_name, email, role_name, affiliations): contact = ( Contact.objects.filter( first_name__icontains=first_name, last_name__icontains=last_name, partners__name__in=affiliations) | Contact.objects.filter( first_name__icontains=first_name, last_name__icontains=last_name, partners__acronym__in=affiliations) ).first() if contact is None: contact = Contact.objects.create( first_name=first_name, last_name=last_name, email=email, type=ContactType.objects.get(name=role_name) ) for affiliation in affiliations: partner = Partner.objects.filter(name=affiliation) if len(partner): contact.partners.add(partner[0]) else: self.logger.warning(f"Cannot link contact '{first_name} {last_name}' to partner. No partner found for the affiliation: {affiliation}") contact.save() return contact