def load_plugins(self): """ Load and activate all IntegrationPlugins """ if not settings.PLUGINS_ENABLED: # Plugins not enabled, do nothing return # pragma: no cover logger.info('Start loading plugins') # Set maintanace mode _maintenance = bool(get_maintenance_mode()) if not _maintenance: set_maintenance_mode(True) registered_successful = False blocked_plugin = None retry_counter = settings.PLUGIN_RETRY while not registered_successful: try: # We are using the db so for migrations etc we need to try this block self._init_plugins(blocked_plugin) self._activate_plugins() registered_successful = True except (OperationalError, ProgrammingError): # pragma: no cover # Exception if the database has not been migrated yet logger.info('Database not accessible while loading plugins') break except IntegrationPluginError as error: logger.error( f'[PLUGIN] Encountered an error with {error.path}:\n{error.message}' ) log_error({error.path: error.message}, 'load') blocked_plugin = error.path # we will not try to load this app again # Initialize apps without any plugins self._clean_registry() self._clean_installed_apps() self._activate_plugins(force_reload=True) # We do not want to end in an endless loop retry_counter -= 1 if retry_counter <= 0: # pragma: no cover if settings.PLUGIN_TESTING: print('[PLUGIN] Max retries, breaking loading') break if settings.PLUGIN_TESTING: print( f'[PLUGIN] Above error occured during testing - {retry_counter}/{settings.PLUGIN_RETRY} retries left' ) # now the loading will re-start up with init # Remove maintenance mode if not _maintenance: set_maintenance_mode(False) logger.info('Finished loading plugins')
def set_maintenance_mode(self, value): try: core.set_maintenance_mode(value) except IOError: raise CommandError( 'Unable to write state file at: %s' % ( settings.MAINTENANCE_MODE_STATE_FILE_NAME, ))
def test_context_manager_override(self): with self.settings(MAINTENANCE_MODE_STATE_FILE_PATH=self.tmp_dir): for environ, override, result in self.override_cases: core.set_maintenance_mode(environ) with core.override_maintenance_mode(override): self.assertEqual(core.get_maintenance_mode(), result) self.assertEqual(core.get_maintenance_mode(), environ)
def maintenance_mode_off(request): if request.user.is_superuser: core.set_maintenance_mode(False) return HttpResponseRedirect('/')
def __reset_state(self): settings.MAINTENANCE_MODE = None core.set_maintenance_mode(False) try: os.remove(settings.MAINTENANCE_MODE_STATE_FILE_PATH) except OSError: pass
def test_context_manager_off(self): with self.settings(MAINTENANCE_MODE_STATE_FILE_PATH=self.tmp_dir): for value in [True, False]: core.set_maintenance_mode(value) with core.maintenance_mode_off(): self.assertEqual(core.get_maintenance_mode(), False) self.assertEqual(core.get_maintenance_mode(), value)
def test_core_maintenance_disabled(self): # Test `get_maintenance_mode` returns maintenance mode from settings - disabled self.__reset_state() core.set_maintenance_mode(True) # Enable maintenance mode in lock file settings.MAINTENANCE_MODE = False val = core.get_maintenance_mode() self.assertFalse(val)
def test_context_processor(self): with self.settings(MAINTENANCE_MODE_STATE_FILE_PATH=self.tmp_dir): for environ, override, result in self.override_cases: core.set_maintenance_mode(environ) with core.override_maintenance_mode(override): self.assertEqual(core.get_maintenance_mode(), result)
def ready(self): if settings.PLUGINS_ENABLED: if not canAppAccessDatabase(allow_test=True): logger.info("Skipping plugin loading sequence") # pragma: no cover else: logger.info('Loading InvenTree plugins') if not registry.is_loading: # this is the first startup try: from common.models import InvenTreeSetting if InvenTreeSetting.get_setting('PLUGIN_ON_STARTUP', create=False): # make sure all plugins are installed registry.install_plugin_file() except: # pragma: no cover pass # get plugins and init them registry.collect_plugins() registry.load_plugins() # drop out of maintenance # makes sure we did not have an error in reloading and maintenance is still active set_maintenance_mode(False) # check git version registry.git_is_modern = check_git_version() if not registry.git_is_modern: # pragma: no cover # simulating old git seems not worth it for coverage log_error(_('Your enviroment has an outdated git version. This prevents InvenTree from loading plugin details.'), 'load') else: logger.info("Plugins not enabled - skipping loading sequence") # pragma: no cover
def handle(self, *args, **options): if django.VERSION < (1, 8): if len(args) != 1: raise CommandError('Error: expected 1 argument: %s' % (self.args, )) state = args[0] else: state = options['state'] state = state.lower() if state in ['on', 'yes', 'true', '1']: core.set_maintenance_mode(True) elif state in ['off', 'no', 'false', '0']: core.set_maintenance_mode(False) else: raise CommandError('Error: invalid argument: \'%s\' ' 'expected %s' % ( state, self.args, ))
def unload_plugins(self): """ Unload and deactivate all IntegrationPlugins """ if not settings.PLUGINS_ENABLED: # Plugins not enabled, do nothing return # pragma: no cover logger.info('Start unloading plugins') # Set maintanace mode _maintenance = bool(get_maintenance_mode()) if not _maintenance: set_maintenance_mode(True) # pragma: no cover # remove all plugins from registry self._clean_registry() # deactivate all integrations self._deactivate_plugins() # remove maintenance if not _maintenance: set_maintenance_mode(False) # pragma: no cover logger.info('Finished unloading plugins')
def test_core_maintenance_disabled(self): self.__reset_state() core.set_maintenance_mode(True) settings.MAINTENANCE_MODE = False val = core.get_maintenance_mode() self.assertFalse(val)
def sf_maintenance_mode(request, key, state): if not key == KEY: exit() if request.method == "POST": if state == "0": set_maintenance_mode(True) else: set_maintenance_mode(False) return redirect("sf_maintenace", key=key)
def test_context_processor(self): self.__reset_state() core.set_maintenance_mode(True) response = self.client.get('/') val = response.context.get('maintenance_mode', False) self.assertTrue(val) core.set_maintenance_mode(False)
def maintenance_mode_off(request): """ Deactivate maintenance-mode and redirect to site root. Only superusers are allowed to use this view. """ if request.user.is_superuser: set_maintenance_mode(False) return HttpResponseRedirect('/')
def ready(self): if not plugins.is_loading: # this is the first startup plugins.collect_plugins() plugins.load_plugins() # drop out of maintenance # makes sure we did not have an error in reloading and maintenance is still active set_maintenance_mode(False)
def test_decorator(self): with self.settings(MAINTENANCE_MODE_STATE_FILE_PATH=self.tmp_dir): for environ, override, result in self.override_cases: core.set_maintenance_mode(environ) @core.override_maintenance_mode(override) def test_function(): self.assertEqual(core.get_maintenance_mode(), result) test_function()
def test_core(self): self.__reset_state() core.set_maintenance_mode(True) val = core.get_maintenance_mode() self.assertTrue(val) core.set_maintenance_mode(False) val = core.get_maintenance_mode() self.assertFalse(val)
def maintenance_mode_off(request): """ Deactivate maintenance-mode and redirect to site root. Only superusers are allowed to use this view. """ if request.user.is_superuser: set_maintenance_mode(False) redirect_to = request.META.get('SCRIPT_NAME', '/') return HttpResponseRedirect('{}/'.format(redirect_to) if not redirect_to.endswith('/') else redirect_to)
def handle(self, *args, **options): if len(args) > 0: arg_value = args[0].lower() if arg_value in ['on', 'yes', 'true', '1']: core.set_maintenance_mode(True) return elif arg_value in ['off', 'no', 'false', '0']: core.set_maintenance_mode(False) return print(self.help)
def handle(self, *args, **options): verbosity = int(options['verbosity']) verbose = True if verbosity == 3 else False if django.VERSION < (1, 8): if len(args) != 1: raise CommandError('Error: expected 1 argument: %s' % (self.args, )) state = args[0] else: state = options['state'] state = state.lower() value = core.get_maintenance_mode() if state in ['on', 'yes', 'true', '1']: if value: if verbose: self.stdout.write('maintenance mode is already on') return if verbose: if self.confirm('maintenance mode on? (y/N) '): core.set_maintenance_mode(True) else: core.set_maintenance_mode(True) elif state in ['off', 'no', 'false', '0']: if not value: if verbose: self.stdout.write('maintenance mode is already off') return if verbose: if self.confirm('maintenance mode off? (y/N) '): core.set_maintenance_mode(False) else: core.set_maintenance_mode(False) else: raise CommandError('Error: invalid argument: \'%s\' ' 'expected %s' % ( state, self.args, )) if verbose: output = 'maintenance mode: %s' % ( 'on' if core.get_maintenance_mode() else 'off', ) self.stdout.write(output) return
def unload_plugins(self): """unload and deactivate all IntegrationPlugins""" logger.info('Start unloading plugins') # set maintanace mode _maintenance = bool(get_maintenance_mode()) if not _maintenance: set_maintenance_mode(True) # remove all plugins from registry self._clean_registry() # deactivate all integrations self._deactivate_plugins() # remove maintenance if not _maintenance: set_maintenance_mode(False) logger.info('Finished unloading plugins')
def import_coursedefs(year, csv, encoding, email=None, delete=True): """ Imports the course definitions for a specific year (Brio/Hyperion export). :param year: the year to import the results for (eg 2015) :type year: int :param csv: the CSV file to import, can be gzip compressed :type csv: str :param encoding: the file encoding (eg utf-8) :type encoding: str :param email: the (optional) email address to send a notification to :type email: str :param delete: whether to delete the data file :type delete: bool :return: None if successful, otherwise error message :rtype: str """ result = None set_maintenance_mode(True) # delete previous rows for year CourseDefs.objects.all().filter(year=year).delete() # import try: csvfile = open(csv, encoding=encoding) reader = DictReader(csvfile) reader.fieldnames = [ name.lower().replace(" ", "_") for name in reader.fieldnames ] count = 0 for row in reader: count += 1 truncate_strings(row, 250) encode_strings(row, 'utf-8') r = CourseDefs() r.year = year r.code = string_cell(row, ['papercode']) r.title = string_cell(row, ['papertitle']) r.description = string_cell(row, ['paperdescription']) r.type = string_cell(row, ['papertype']) r.stage = int_cell(row, ['paperstage']) r.points = float_cell(row, ['paperpoints']) r.delivery_mode = string_cell(row, ['paperdeliverymode']) r.owning_programme = string_cell(row, ['paperowningprogramme']) r.owning_programme_title = string_cell( row, ['paperowningprogrammetitle']) r.fw_level = int_cell(row, ['paperfwlevel']) r.hours_contact = int_cell(row, ['paperhourscontact']) r.hours_self_directed = int_cell(row, ['paperhoursselfdirected']) r.hours_other_directed = int_cell(row, ['paperhoursotherdirected']) r.funding_source = string_cell(row, ['paperfundingsource']) r.course_factor = float_cell(row, ['papercoursefactor']) r.cost_category_code = string_cell(row, ['papercostcategorycode']) r.cost_category = string_cell(row, ['papercostcategory']) r.funding_class_code = string_cell(row, ['paperfundingclasscode']) r.funding_class = string_cell(row, ['paperfundingclass']) r.individual_efts = int_cell(row, ['paperindividualefts']) r.nzsced_code = string_cell(row, ['nzscedcode']) r.nzsced_category = string_cell(row, ['nzscedcategory']) r.delivering_school_code = string_cell( row, ['paperdeliveringschoolcode']) r.delivering_school = string_cell(row, ['paperdeliveringschool']) r.delivering_dept_code = string_cell(row, ['paperdeliveringdeptcode']) r.delivering_dept = string_cell(row, ['paperdeliveringdept']) r.delivering_unit_code = string_cell(row, ['paperdeliveringunitcode']) r.delivering_unit = string_cell(row, ['paperdeliveringunit']) r.owning_school_code = string_cell(row, ['paperowningschoolcode']) r.owning_school = string_cell(row, ['paperowningschool']) r.owning_dept_code = string_cell(row, ['paperowningdepartmentcode']) r.owning_dept = string_cell(row, ['paperowningdepartment']) r.owning_unit_code = string_cell(row, ['paperowningunitcode']) r.owning_unit = string_cell(row, ['paperowningunit']) r.self_paced = bool_cell(row, ['papertitle']) r.online = bool_cell(row, ['paperonline']) r.active = bool_cell(row, ['paperactive']) r.pending = bool_cell(row, ['paperpending']) r.sub_status = string_cell(row, ['papersubstatus']) r.grade_method_code = string_cell(row, ['grademethodcode']) r.pbrf_eligibility = string_cell(row, ['pbrfeligibility']) r.coe_policy = string_cell(row, ['coepolicy']) r.report_academic_result = bool_cell(row, ['reportacademicresult']) r.internet_based = string_cell(row, ['paperinternetbased']) r.save() # progress if (count % 1000) == 0: update_tablestatus(CourseDefs._meta.db_table, "Imported " + str(count) + " rows...") # close file csvfile.close() except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) return msg finally: if delete: try: os.remove(csv) except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg if email is not None: send_email( email, 'Import: course definitions', 'Import succeeded' if (result is None) else 'Import failed: ' + result) return result
def import_supervisors(csv, encoding, email=None, delete=True): """ Imports the supervisors (Jade Export). :param csv: the CSV file to import :type csv: str :param encoding: the file encoding (eg utf-8) :type encoding: str :param email: the (optional) email address to send a notification to :type email: str :param delete: whether to delete the data file :type delete: bool :return: None if successful, otherwise error message :rtype: str """ result = None set_maintenance_mode(True) # empty table Supervisors.objects.all().delete() # import p1 = re.compile('.*\/') p2 = re.compile(' .*') try: with open(csv, encoding=encoding) as csvfile: reader = DictReader(csvfile) reader.fieldnames = [ name.lower().replace(" ", "_") for name in reader.fieldnames ] count = 0 for row in reader: count += 1 truncate_strings(row, 250) encode_strings(row, 'utf-8') r = Supervisors() r.student_id = row['student'][ row['student'].rfind(' ') + 1:] # extract ID at end of string r.student = row['student'] r.supervisor = row['supervisor'] r.active_roles = row['active_roles'] r.entity = row['entity'] r.agreement_status = row['agreement_status'] r.date_agreed = parse_supervisors_date('date_agreed', row['date_agreed']) r.completion_date = parse_supervisors_date( 'completion_date', row['completion_date']) r.proposed_enrolment_date = parse_supervisors_date( 'proposed_enrolment_date', row['proposed_enrolment_date']) r.proposed_research_topic = row['proposed_research_topic'] # normalize title a bit title = row['title'] title = title.lower() title = title.replace(".", "").replace("/", "").replace(" ", "") title = title.replace("associate", "a").replace("assoc", "a") title = title.replace("professor", "prof").replace( "pro", "prof").replace("proff", "prof") title = title.replace("doctor", "dr") title = title.replace("sir", "") r.title = title r.quals = row['quals'] r.comments = row['comments'] # active if not withdrawn r.active = ("removed" not in title) and ( "replaced" not in title) and ("informal" not in title) # determine program type program = p2.sub('', p1.sub('', row['entity'])).upper() r.program = award_to_program(program) r.save() # progress if (count % 1000) == 0: update_tablestatus(Supervisors._meta.db_table, "Imported " + str(count) + " rows...") except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg finally: if delete: try: os.remove(csv) except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg if email is not None: send_email( email, 'Import: supervisors', 'Import succeeded' if (result is None) else 'Import failed: ' + result) return result
def populate_student_dates(email=None): """ Populates the studentdates table. Truncates the table first. :param email: the (optional) email address to send a notification to :type email: str :return: None if successful, otherwise error message :rtype: str """ set_maintenance_mode(True) # delete old rows StudentDates.objects.all().delete() # get all students that are supervised table_super = Supervisors._meta.db_table cursor = connection.cursor() cursor.execute(""" SELECT DISTINCT(student_id) FROM %s """ % table_super) table = GradeResults._meta.db_table cursor2 = connection.cursor() count = 0 for row in cursor.fetchall(): sid = str(row[0]).strip() count += 1 logger.debug(str(count) + "/studentid: " + sid) master_months = None master_start = None master_end = None master_school = '' master_dept = '' master_fulltime = None master_status = None phd_months = None phd_start = None phd_end = None phd_school = '' phd_dept = '' phd_fulltime = None phd_status = None try: # master - months sql = """ select sum(credits / cast(regexp_replace(paper_occurrence, '([A-Z]+)59([3456789])-(.*)', '\\2') as integer) / 30 * 12), count(*) from %s where student_id = '%s' and programme_type_code = 'MD' and paper_occurrence ~ '([A-Z]+)59([3456789])-(.*)' group by student_id """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): master_months = row2[0] break # master - start date sql = """ select min(occurrence_startdate), owning_school_clevel, owning_department_clevel from %s where student_id = '%s' and programme_type_code = 'MD' and paper_occurrence ~ '([A-Z]+)59([3456789])-(.*)' group by student_id, owning_school_clevel, owning_department_clevel """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): master_start = row2[0] master_school = row2[1] master_dept = row2[2] break # master - end date sql = """ select max(occurrence_enddate) from %s where student_id = '%s' and programme_type_code = 'MD' and paper_occurrence ~ '([A-Z]+)59([3456789])-(.*)' and final_grade is not null group by student_id """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): master_end = row2[0] break # master - full time sql = """ select avg(credits) >= %d from %s where student_id = '%s' and programme_type_code = 'MD' """ % (FULL_TIME_CREDITS, table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): master_fulltime = row2[0] break # master - status sql = """ select student_id, name, year, final_grade_status, final_grade from %s where student_id = '%s' and programme_type_code = 'MD' and paper_occurrence ~ '([A-Z]+)59([3456789])-(.*)' order by year desc """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): master_status = row2[3] if master_status == "": master_status = None if master_status == "C": master_status = "finished" if master_status == "N": master_status = "current" if row2[4] == "WD": master_status = "withdrawn" break # PhD - months sql = """ select sum(coalesce(student_credit_points, credits_per_student) / credits) * 12, count(*) from %s where student_id = '%s' and programme_type_code = 'DP' group by student_id """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): phd_months = row2[0] break # PhD - start date (1) sql = """ select min(occurrence_startdate), owning_school_clevel, owning_department_clevel from %s where student_id = '%s' and programme_type_code = 'DP' group by student_id, owning_school_clevel, owning_department_clevel """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): phd_start = row2[0] phd_school = row2[1] phd_dept = row2[2] break # PhD - start date (2) sql = """ select proposed_enrolment_date from %s where student_id = '%s' and active = 'true' and program = 'DP' """ % (table_super, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): if (phd_start is None) or (row2[0] > phd_start): phd_start = row2[0] break # PhD - end date (1) sql = """ select completion_date from %s where student_id = '%s' and active = 'true' and program = 'DP' and completion_date > '1900-01-01' """ % (table_super, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): phd_end = row2[0] break # PhD - end date (2) if phd_end is None: sql = """ select occurrence_enddate from %s where student_id = '%s' and programme_type_code = 'DP' order by occurrence_enddate desc """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): phd_end = row2[0] break # PhD - full time sql = """ select avg(credits_per_student) >= %d from %s where student_id = '%s' and programme_type_code = 'DP' """ % (FULL_TIME_CREDITS, table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): phd_fulltime = row2[0] break # PhD - status phd_status = "current" sql = """ select student_id, name, year, final_grade_status, final_grade from %s where student_id = '%s' and programme_type_code = 'DP' and final_grade != '...' and final_grade != '' order by year desc """ % (table, sid) cursor2.execute(sql) for row2 in cursor2.fetchall(): phd_status = row2[3] if phd_status == "": phd_status = None if phd_status == "C": phd_status = "finished" if phd_status == "N": phd_status = "current" if row2[4] == "WD": phd_status = "withdrawn" break # save if phd_start is not None: r = StudentDates() r.student_id = sid r.program = "DP" r.start_date = phd_start r.end_date = phd_end if phd_end is not None else '9999-12-31' r.months = phd_months r.school = phd_school r.department = phd_dept r.full_time = phd_fulltime r.status = phd_status r.save() if master_start is not None: r = StudentDates() r.student_id = sid r.program = "MD" r.start_date = master_start r.end_date = master_end if master_end is not None else '9999-12-31' r.months = master_months r.school = master_school r.department = master_dept r.full_time = master_fulltime r.status = master_status r.save() except Exception as ex: msg = "PhD: id=%s, start=%s, end=%s, months=%s" % ( sid, phd_start, phd_end, phd_months) msg += "\n" + "Master: id=%s, start=%s, end=%s, months=%s" % ( sid, master_start, master_end, master_months) logger.exception(msg=msg) # progress if (count % 1000) == 0: update_tablestatus(StudentDates._meta.db_table, "Processed " + str(count) + " students...") set_maintenance_mode(False) if email is not None: send_email(email, 'Student dates', 'Finished calculating dates') return None
def import_scholarships(csv, encoding, email=None, delete=True): """ Imports the scholarships (Jade Export). :param csv: the CSV file to import :type csv: str :param encoding: the file encoding (eg utf-8) :type encoding: str :param email: the (optional) email address to send a notification to :type email: str :param delete: whether to delete the data file :type delete: bool :return: None if successful, otherwise error message :rtype: str """ result = None set_maintenance_mode(True) # empty table Scholarship.objects.all().delete() # import try: with open(csv, encoding=encoding) as csvfile: reader = DictReader(csvfile) reader.fieldnames = [ name.lower().replace(" ", "_") for name in reader.fieldnames ] count = 0 for row in reader: count += 1 truncate_strings(row, 250) encode_strings(row, 'utf-8') r = Scholarship() r.student_id = row['person_id'] r.name = row['template'] r.status = row['status'] r.decision = row['decision'] r.year = int(row['year']) r.save() # progress if (count % 1000) == 0: update_tablestatus(Scholarship._meta.db_table, "Imported " + str(count) + " rows...") except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg finally: if delete: try: os.remove(csv) except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg if email is not None: send_email( email, 'Import: scholarships', 'Import succeeded' if (result is None) else 'Import failed: ' + result) return result
def import_grade_results(year, csv, isgzip, encoding, email=None, delete=True): """ Imports the grade results for a specific year (Brio/Hyperion export). :param year: the year to import the results for (eg 2015) :type year: int :param csv: the CSV file to import, can be gzip compressed :type csv: str :param isgzip: true if GZIP compressed :type isgzip: bool :param encoding: the file encoding (eg utf-8) :type encoding: str :param email: the (optional) email address to send a notification to :type email: str :param delete: whether to delete the data file :type delete: bool :return: None if successful, otherwise error message :rtype: str """ result = None set_maintenance_mode(True) query_date = None # delete previous rows for year GradeResults.objects.all().filter(year=year).delete() # import try: if isgzip: csvfile = gzip.open(csv, mode='rt', encoding=encoding) else: csvfile = open(csv, encoding=encoding) reader = DictReader(csvfile) reader.fieldnames = [ name.lower().replace(" ", "_") for name in reader.fieldnames ] count = 0 for row in reader: count += 1 truncate_strings(row, 250) encode_strings(row, 'utf-8') r = GradeResults() r.year = year r.student_id = string_cell(row, ['student_id']) r.name = string_cell(row, ['name']) r.title = string_cell(row, ['title']) r.prefered_given_name = string_cell(row, ['prefered_given_name'], defvalue='') r.given_name = string_cell(row, ['given_name'], defvalue='') r.other_given_names = string_cell(row, ['other_given_names'], defvalue='') r.family_name = string_cell(row, ['family_name'], defvalue='') r.previous_name = string_cell(row, ['previous_name']) r.address1 = string_cell(row, ['address1', 'address_line_1']) r.address2 = string_cell(row, ['address2', 'address_line_2']) r.address2a = string_cell(row, ['address2a']) r.address2b = string_cell(row, ['address2b']) r.address3 = string_cell(row, ['address3', 'address_line_3']) r.address4 = string_cell(row, ['address4', 'address_line_4']) r.postcode = string_cell(row, ['postcode', 'postal_area_code']) r.telephone = string_cell(row, ['telephone', 'perm_phone_number']) r.cellphone = string_cell(row, ['cellphone', 'perm_cellphone_number']) r.email = string_cell(row, ['email', 'perm_email_address']) r.hasdisability = int_cell(row, ['hasdisability']) r.isdomestic = int_cell(row, ['isdomestic', 'domestic_indicator']) r.is_domiciled_locally = int_cell(row, ['is_domiciled_locally']) r.citizenship = string_cell(row, ['citizenship']) r.residency_status = string_cell(row, ['residency_status']) r.origin = string_cell(row, ['origin']) r.gender = string_cell(row, ['gender']) r.ethnicity = string_cell(row, ['ethnicity']) r.ethnic_group = string_cell(row, ['ethnic_group']) r.all_ethnicities_string = string_cell(row, ['all_ethnicities_string']) r.all_iwi_string = string_cell(row, ['all_iwi_string']) r.dateofbirth = parse_grade_results_date( 'dateofbirth', string_cell(row, ['dateofbirth', 'date_of_birth'])) r.dateofdeath = string_cell(row, ['dateofdeath']) r.waikato_1st = int_cell(row, ['waikato_1st']) r.nz_1st = int_cell(row, ['nz_1st']) r.last_year_sec = int_cell(row, ['last_year_sec']) r.sec_qual_year = int_cell(row, ['sec_qual_year']) r.last_sec_school = string_cell(row, ['last_sec_school']) r.last_sec_school_region = string_cell(row, ['last_sec_school_region']) r.highest_sec_qual = string_cell(row, ['highest_sec_qual']) r.main_activity = string_cell(row, ['main_activity']) r.award_title = string_cell(row, ['award_title', 'award']) r.prog_abbr = string_cell(row, ['prog_abbr', 'prog_-_abbr']) r.programme = string_cell(row, ['programme']) r.programme_type_code = string_cell(row, ['programme_type_code']) r.programme_type = string_cell(row, ['programme_type']) r.ishigherdegree = int_cell(row, ['ishigherdegree']) r.school_of_study = string_cell(row, ['school_of_study']) r.school_of_study_clevel = fix_org_unit( string_cell(row, ['school_of_study_clevel'])) r.paper_master_code = string_cell( row, ['paper_master_code', 'paper_master']) r.paper_occurrence = string_cell(row, ['paper_occurrence']) r.paper_title = string_cell(row, ['paper_title']) r.occurrence_startdate = parse_grade_results_date( 'occurrence_startdate', string_cell(row, ['occurrence_startdate'])) r.occurrence_startyear = int_cell(row, ['occurrence_startyear']) r.occurrence_startweek = int_cell(row, ['occurrence_startweek']) r.occurrence_enddate = parse_grade_results_date( 'occurrence_enddate', string_cell(row, ['occurrence_enddate'])) r.stage = int_cell(row, ['stage']) r.credits = float_cell(row, ['credits']) r.student_credit_points = float_cell( row, ['student_credit_points', 'student_credits']) r.iscancelled = int_cell(row, ['iscancelled']) r.isoncampus = int_cell(row, ['isoncampus']) r.issemesteracourse = int_cell(row, ['issemesteracourse']) r.issemesterbcourse = int_cell(row, ['issemesterbcourse']) r.iswholeyearcourse = int_cell(row, ['iswholeyearcourse']) r.location_code = string_cell(row, ['location_code']) r.location = string_cell(row, ['location']) r.owning_school_clevel = fix_org_unit( string_cell(row, ['owning_school_clevel'])) r.owning_school = string_cell(row, ['owning_school']) r.owning_department_clevel = string_cell( row, ['owning_department_clevel']) r.owning_department = string_cell(row, ['owning_department']) r.owning_level4_clevel = string_cell(row, ['owning_level4_clevel']) r.owning_level4_department = string_cell( row, ['owning_level4_department']) r.owning_level4or3_department = string_cell( row, ['owning_level4or3_department']) r.owning_level4or3_clevel = string_cell( row, ['owning_level4or3_clevel']) r.delivery_mode_code = string_cell(row, ['delivery_mode_code']) r.delivery_mode = string_cell(row, ['delivery_mode']) r.semester_code = string_cell(row, ['semester_code']) r.semester_description = string_cell(row, ['semester_description']) r.isselfpaced = int_cell(row, ['isselfpaced']) r.source_of_funding = string_cell(row, ['source_of_funding']) r.funding_category_code = string_cell(row, ['funding_category_code']) r.funding_category = string_cell(row, ['funding_category']) r.cost_category_code = string_cell(row, ['cost_category_code']) r.cost_category = string_cell(row, ['cost_category']) r.research_supplement_code = int_cell(row, ['research_supplement_code']) r.research_supplement = string_cell(row, ['research_supplement']) r.classification_code = float_cell(row, ['classification_code']) r.classification = string_cell(row, ['classification']) r.division = string_cell(row, ['division']) r.division_code = string_cell(row, ['division_code']) r.specified_programme = string_cell(row, ['specified_programme']) r.major = string_cell(row, ['major']) r.second_major = string_cell(row, ['second_major']) r.major2 = string_cell(row, ['major2']) r.second_major2 = string_cell(row, ['second_major2']) r.main_subject = string_cell(row, ['main_subject']) r.second_subject = string_cell(row, ['second_subject']) r.supporting_subject = string_cell(row, ['supporting_subject']) r.teaching_1 = string_cell(row, ['teaching_1']) r.teaching_2 = string_cell(row, ['teaching_2']) r.teaching_3 = string_cell(row, ['teaching_3']) r.teaching_4 = string_cell(row, ['teaching_4']) r.subject = string_cell(row, ['subject']) r.field = string_cell(row, ['field']) r.specialisation = string_cell(row, ['specialisation']) r.stream = string_cell(row, ['stream']) r.endorsement = string_cell(row, ['endorsement']) r.award_year = int_cell(row, ['award_year']) r.award_completion_status = string_cell( row, ['award_completion_status']) r.award_completion_date = parse_grade_results_date( 'award_completion_date', string_cell(row, ['award_completion_date'])) r.award_completion_confirmed_date = parse_grade_results_date( 'award_completion_confirmed_date', string_cell(row, ['award_completion_confirmed_date'])) r.admission_year = int_cell(row, ['admission_year']) r.admission_reason = string_cell(row, ['admission_reason']) r.admission_criteria = string_cell(row, ['admission_criteria']) r.admission_status = string_cell(row, ['admission_status']) r.grade = string_cell(row, ['grade']) r.grade_status = string_cell(row, ['grade_status']) r.result_status_code = string_cell(row, ['result_status_code']) r.result_status = string_cell(row, ['result_status']) r.grade_ranking = int_cell(row, ['grade_ranking']) r.mark = float_cell(row, ['mark']) r.moe_completion_code = int_cell(row, ['moe_completion_code']) r.iscontinuinggrade = int_cell(row, ['iscontinuinggrade']) r.ispassgrade = int_cell(row, ['ispassgrade']) r.query_date = parse_grade_results_date( 'query_date', string_cell(row, ['query_date'])) if (query_date is None) and (r.query_date is not None): query_date = datetime.strptime( parse_grade_results_date('query_date', string_cell(row, ['query_date'])), "%Y-%m-%d") r.enr_year = int_cell(row, ['enr_year', 'enrolment_year']) r.enrolment_status = string_cell(row, ['enrolment_status']) r.final_grade = string_cell(row, ['final_grade']) r.final_grade_ranking = int_cell(row, ['final_grade_ranking']) r.final_grade_status = string_cell(row, ['final_grade_status']) r.final_grade_result_status = string_cell( row, ['final_grade_result_status']) r.papers_per_student = int_cell(row, ['papers_per_student']) r.credits_per_student = float_cell(row, ['credits_per_student']) r.gpa = float_cell(row, ['gpa']) r.ones = int_cell(row, ['ones']) r.allgradeones = int_cell(row, ['allgradeones']) r.passgradeones = int_cell(row, ['passgradeones']) r.retentionones = int_cell(row, ['retentionones']) r.award_completion_year = int_cell(row, ['award_completion_year']) r.personoid = float_cell(row, ['personoid']) r.courseoccurrenceoid = float_cell(row, ['courseoccurrenceoid']) r.awardenrolmentoid = float_cell(row, ['awardenrolmentoid']) r.enrolmentorcosuoid = float_cell(row, ['enrolmentorcosuoid']) r.isformalprogramme = int_cell(row, ['isformalprogramme']) r.citizenship_simple = string_cell( row, ['citizenship_simple', 'citizenship_code']) r.moe_pbrf_code = string_cell(row, ['moe_pbrf_code']) r.moe_pbrf = string_cell(row, ['moe_pbrf']) r.achievement_date = parse_grade_results_date( 'achievement_date', string_cell(row, ['achievement_date'])) r.te_reo = int_cell(row, ['te_reo']) r.save() # progress if (count % 1000) == 0: update_tablestatus(GradeResults._meta.db_table, "Imported " + str(count) + " rows...") # close file csvfile.close() except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) return msg finally: if delete: try: os.remove(csv) except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg # if (result is None) and (query_date is not None) and (query_date.year == datetime.today().year): update_tablestatus(GradeResults._meta.db_table, timestamp=query_date) if email is not None: send_email( email, 'Import: grade results', 'Import succeeded' if (result is None) else 'Import failed: ' + result) return result
def import_bulk(csv, email=None): """ Performs a bulk import. The CSV file has to have the following layout (headers must match!): type,year,file,isgzip,encoding graderesults,2007,/some/where/2007.csv.gz,True,iso-8859-1 graderesults,2006,/some/where/2006.csv,False,iso-8859-1 scholarships,,/some/where/scholarships.csv,False,iso-8859-1 supervisors,,/some/where/supervisors.csv,False,iso-8859-1 ... :param csv: the CSV file with the files to bulk import :type csv: str :param email: the (optional) email address to send a notification to :type email: str :return: None if successful, otherwise error message :rtype: str """ result = [] set_maintenance_mode(True) try: csvfile = open(csv) reader = DictReader(csvfile) for row in reader: if len(row) != 5: continue try: logger.debug("Importing: " + str(row)) if row['type'] == 'graderesults': msg = import_grade_results(int(row['year']), row['file'], (row['isgzip'] == 'True'), row['encoding'], delete=False) if msg is None: update_tablestatus(GradeResults._meta.db_table) else: result.append(msg) elif row['type'] == 'coursdefs': msg = import_grade_results(int(row['year']), row['file'], row['encoding'], delete=False) if msg is None: update_tablestatus(CourseDefs._meta.db_table) else: result.append(msg) elif row['type'] == 'supervisors': msg = import_supervisors(row['file'], row['encoding'], delete=False) if msg is None: update_tablestatus(Supervisors._meta.db_table) else: result.append(msg) elif row['type'] == 'scholarships': msg = import_scholarships(row['file'], row['encoding'], delete=False) if msg is None: update_tablestatus(Scholarship._meta.db_table) else: result.append(msg) elif row['type'] == 'associatedrole': msg = import_associatedrole(row['file'], row['encoding'], delete=False) if msg is None: update_tablestatus(AssociatedRole._meta.db_table) else: result.append(msg) else: result.append("Unhandled file type: " + str(row['type'])) except Exception as ex: traceback.print_exc(file=sys.stdout) result.append(str(ex)) except Exception as ex: traceback.print_exc(file=sys.stdout) result.append(str(ex)) if len(result) == 0: logger.info("Populating student dates") populate_student_dates() update_tablestatus(StudentDates._meta.db_table) result_msg = None else: result_msg = '\n'.join(result) if email is not None: send_email( email, 'Import: bulk', 'Import succeeded' if (result_msg is None) else 'Import failed: ' + result_msg) return result_msg
def on_maintenance(request): set_maintenance_mode(True) messages.info(request, "Maintenance mode ON!") return HttpResponseRedirect(reverse("admin:index"))
def import_associatedrole(csv, encoding, email=None, delete=True): """ Imports the associated role (Jade Export). :param csv: the CSV file to import :type csv: str :param encoding: the file encoding (eg utf-8) :type encoding: str :param email: the (optional) email address to send a notification to :type email: str :param delete: whether to delete the data file :type delete: bool :return: None if successful, otherwise error message :rtype: str """ result = None set_maintenance_mode(True) # empty table AssociatedRole.objects.all().delete() # import try: with open(csv, encoding=encoding) as csvfile: reader = DictReader(csvfile) reader.fieldnames = [ name.lower().replace(" ", "_") for name in reader.fieldnames ] count = 0 for row in reader: count += 1 truncate_strings(row, 250) encode_strings(row, 'utf-8') r = AssociatedRole() r.role = row['role'] r.person = row['person'] r.entity = row['entity'] r.valid_from = parse_associatedrole_date( 'valid_from', row['valid_from']) r.valid_to = parse_associatedrole_date('valid_to', row['valid_to']) r.active = len(row['valid_to'].strip()) == 0 if " - " in r.entity: r.student_id = r.entity[(r.entity.index(" - ") + 3):] if "Award/" in r.entity: r.student = r.entity[r.entity.index("Award/") + 6:] r.program = award_to_program( r.student[0:r.student.index(" ")].upper()) r.student = r.student[r.student.index(" ") + 1:] r.save() # progress if (count % 1000) == 0: update_tablestatus(AssociatedRole._meta.db_table, "Imported " + str(count) + " rows...") except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg finally: if delete: try: os.remove(csv) except Exception as ex: msg = traceback.format_exc() logger.error(msg=msg) result = msg if email is not None: send_email( email, 'Import: associated role', 'Import succeeded' if (result is None) else 'Import failed: ' + result) return result