def equalize_dbs( taxpayers, shared_variables ): # Taxpayers are the ones splitted for this specific subprocess try: # Shared variables: current_taxpayer = shared_variables['current_taxpayer'] total_taxpayers = shared_variables['total_taxpayers'] current_table_row = shared_variables['current_table_row'] forcing_execution = shared_variables['forcing_execution'] process_params = shared_variables['process_params'] forcing_period = False if 'year' in process_params or 'all' in process_params: forcing_period = True lock = shared_variables['lock'] # Process: process_name = multiprocessing.current_process().name total_taxpayers_for_this_subprocess = len(taxpayers) # Process logger: process_logger = _Utilities.get_subprocess_logger( process_name, EQUALIZATION_LOGGING_CONFIG, logger=equalization_logger) process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(_Constants.LOG_SEPARATOR) process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' + process_name.upper()) taxpayers_equalized_counter = 0 process_logger.info(LOG_INDENT + 'Forcing execution: ' + str(forcing_execution)) process_logger.info(LOG_INDENT + 'Taxpayers: ' + str(total_taxpayers_for_this_subprocess)) for taxpayer in taxpayers: if not forcing_execution: _Utilities.update_current_taxpayer(_Constants.EQUALIZATION, taxpayer['identifier'], current_taxpayer.value + 1, logger=process_logger) percentage_of_equalization_done = _Utilities.get_process_percentage_done( taxpayers_equalized_counter, total_taxpayers_for_this_subprocess) taxpayers_equalized_counter = taxpayers_equalized_counter + 1 # Specific taxpayers (this thread's counter) process_logger.info(LOG_INDENT + '-> (' + str(taxpayers_equalized_counter) + '/' + str(total_taxpayers_for_this_subprocess) + ') ' + taxpayer['identifier'] + ' --- ' + percentage_of_equalization_done) equalization_data = equalize_dbs_for_a_taxpayer( forcing_period=forcing_period, forcing_execution=forcing_execution, taxpayer=taxpayer, process_logger=process_logger, process_params=process_params) with lock: current_taxpayer.value = current_taxpayer.value + 1 current_date = Datetime.now() equalization_log = { 'date': str(current_date)[:10], 'hour': str(current_date)[10:-7], 'process_name': process_name, 'current_taxpayer_index': current_taxpayer.value, 'total_taxpayers': total_taxpayers.value, 'identifier': taxpayer['identifier'], 'forest_db': equalization_data['before']['forest_db'], 'corebook_db': equalization_data['before']['corebook_db'], 'missing_in_forest_db': equalization_data['before']['cb_but_not_in_f'], 'missing_in_cb_db': equalization_data['before']['f_but_not_in_cb'], 'stored': equalization_data['after']['stored'], 'errors': equalization_data['after']['errors'], # 'forest_db' : 90, # 'corebook_db' : 10, # 'missing_in_forest_db' : 10, # 'missing_in_cb_db' : 11, # 'stored' : 10, # 'errors' : 1, 'current_table_row': current_table_row, 'lock': lock } # End of equalization_log if equalization_log['current_taxpayer_index'] == equalization_log[ 'total_taxpayers']: equalization_log['end'] = True equalization_log[ 'end_message'] = EQUALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/' else: equalization_log['end'] = False _Locals.log_eq_thread_logs_at_equalization_main_logs( equalization_execution_log=equalization_log, equalization_logger=equalization_logger, cron_logger=cron_logger) process_logger.info(2 * LOG_INDENT + 'Updating taxpayer ... ') _Utilities.update_taxpayer_status(taxpayer, _Constants.EQUALIZATION, logger=process_logger) process_logger.info( 2 * LOG_INDENT + 'Equalized successfully. Logged at Equalization main logs') process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' + process_name.upper() + ' DONE SUCCESSFULLY \0/') process_logger.info(_Constants.LOG_SEPARATOR) return 'OK' except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: equalization_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def excute_initialization( taxpayers, shared_variables ): # Taxpayers are the ones splitted for this specific subprocess try: # Shared variables: current_taxpayer = shared_variables['current_taxpayer'] total_taxpayers = shared_variables['total_taxpayers'] current_table_row = shared_variables['current_table_row'] forcing_execution = shared_variables['forcing_execution'] process_params = shared_variables['process_params'] forcing_period = False if 'year' in process_params: forcing_period = True lock = shared_variables['lock'] # Process: process_name = multiprocessing.current_process().name total_taxpayers_for_this_subprocess = len(taxpayers) # Process logger: process_logger = _Utilities.get_subprocess_logger( process_name, INITIALIZATION_LOGGING_CONFIG, logger=initialization_logger) process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(_Constants.LOG_SEPARATOR) process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' + process_name.upper()) taxpayers_initialized_counter = 0 process_logger.info(LOG_INDENT + 'Forcing execution: ' + str(forcing_execution)) process_logger.info(LOG_INDENT + 'Taxpayers: ' + str(total_taxpayers_for_this_subprocess)) for taxpayer in taxpayers: if not forcing_execution: _Utilities.update_current_taxpayer(_Constants.INITIALIZATION, taxpayer['identifier'], current_taxpayer.value + 1, logger=process_logger) percentage_of_initialization_done = _Utilities.get_process_percentage_done( taxpayers_initialized_counter, total_taxpayers_for_this_subprocess) taxpayers_initialized_counter = taxpayers_initialized_counter + 1 # Specific taxpayers (this thread's counter) process_logger.info(LOG_INDENT + '-> (' + str(taxpayers_initialized_counter) + '/' + str(total_taxpayers_for_this_subprocess) + ') ' + taxpayer['identifier'] + ' --- ' + percentage_of_initialization_done) initialization_execution_data = excute_initialization_for_taxpayer( forcing_period=forcing_period, forcing_execution=forcing_execution, taxpayer=taxpayer, process_logger=process_logger, process_params=process_params) with lock: current_taxpayer.value = current_taxpayer.value + 1 current_date = Datetime.now() initialization_execution_log = { 'date': str(current_date)[:10], 'hour': str(current_date)[10:-7], 'process_name': process_name, 'current_taxpayer_index': current_taxpayer.value, 'total_taxpayers': total_taxpayers.value, 'identifier': taxpayer['identifier'], 'new': initialization_execution_data['new'], 'stored': initialization_execution_data['stored'], 'year_initialized': initialization_execution_data['year_initialized'], 'month_initialized': initialization_execution_data['month_initialized'], 'percentage_initialized': initialization_execution_data['percentage_initialized'], 'initialized': initialization_execution_data['initialized'], # 'new' : 0, # 'stored' : 0, # 'year_initialized' : '2015', # 'month_initialized' : '02', # 'percentage_initialized' : 0, 'lock': lock, 'current_table_row': current_table_row } # End of initialization_execution_log if initialization_execution_log[ 'current_taxpayer_index'] == initialization_execution_log[ 'total_taxpayers']: initialization_execution_log['end'] = True initialization_execution_log[ 'end_message'] = INITIALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/' else: initialization_execution_log['end'] = False _Locals.log_initiliazation_thread_logs_at_initialization_main_logs( initialization_execution_log=initialization_execution_log, initialization_logger=initialization_logger, cron_logger=cron_logger) if forcing_period or ( 'avoid_iteration' in initialization_execution_data and initialization_execution_data['avoid_iteration'] == True): process_logger.info( 2 * LOG_INDENT + 'NOT Updating initialization data for taxpayer ... ') else: process_logger.info( 2 * LOG_INDENT + 'Updating initialization data for taxpayer ... ') _Locals.update_initialization_data_for_taxpayer( taxpayer, initialization_execution_log, logger=process_logger) process_logger.info( 2 * LOG_INDENT + 'Synchronized successfully. Logged at SL1 main logs') process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' + process_name.upper() + ' DONE SUCCESSFULLY \0/') process_logger.info(_Constants.LOG_SEPARATOR) return 'OK' except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: initialization_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def equalize_dbs(taxpayers,shared_variables):# Taxpayers are the ones splitted for this specific subprocess try: # Shared variables: current_taxpayer = shared_variables['current_taxpayer'] total_taxpayers = shared_variables['total_taxpayers'] current_table_row = shared_variables['current_table_row'] forcing_execution = shared_variables['forcing_execution'] process_params = shared_variables['process_params'] forcing_period = False if 'year' in process_params or 'all' in process_params: forcing_period = True lock = shared_variables['lock'] # Process: process_name = multiprocessing.current_process().name total_taxpayers_for_this_subprocess = len(taxpayers) # Process logger: process_logger = _Utilities.get_subprocess_logger(process_name,EQUALIZATION_LOGGING_CONFIG,logger=equalization_logger) process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(_Constants.LOG_SEPARATOR) process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' + process_name.upper()) taxpayers_equalized_counter = 0 process_logger.info(LOG_INDENT + 'Forcing execution: ' + str(forcing_execution)) process_logger.info(LOG_INDENT + 'Taxpayers: ' + str(total_taxpayers_for_this_subprocess)) for taxpayer in taxpayers: if not forcing_execution: _Utilities.update_current_taxpayer(_Constants.EQUALIZATION,taxpayer['identifier'],current_taxpayer.value+1,logger=process_logger) percentage_of_equalization_done = _Utilities.get_process_percentage_done(taxpayers_equalized_counter,total_taxpayers_for_this_subprocess) taxpayers_equalized_counter = taxpayers_equalized_counter + 1# Specific taxpayers (this thread's counter) process_logger.info(LOG_INDENT + '-> (' + str(taxpayers_equalized_counter) + '/' + str(total_taxpayers_for_this_subprocess) + ') ' + taxpayer['identifier'] + ' --- ' + percentage_of_equalization_done) equalization_data = equalize_dbs_for_a_taxpayer(forcing_period=forcing_period,forcing_execution=forcing_execution,taxpayer=taxpayer,process_logger=process_logger,process_params=process_params) with lock: current_taxpayer.value = current_taxpayer.value + 1 current_date = Datetime.now() equalization_log = { 'date' : str(current_date)[:10], 'hour' : str(current_date)[10:-7], 'process_name' : process_name, 'current_taxpayer_index' : current_taxpayer.value, 'total_taxpayers' : total_taxpayers.value, 'identifier' : taxpayer['identifier'], 'forest_db' : equalization_data['before']['forest_db'], 'corebook_db' : equalization_data['before']['corebook_db'], 'missing_in_forest_db' : equalization_data['before']['cb_but_not_in_f'], 'missing_in_cb_db' : equalization_data['before']['f_but_not_in_cb'], 'stored' : equalization_data['after']['stored'], 'errors' : equalization_data['after']['errors'], # 'forest_db' : 90, # 'corebook_db' : 10, # 'missing_in_forest_db' : 10, # 'missing_in_cb_db' : 11, # 'stored' : 10, # 'errors' : 1, 'current_table_row' : current_table_row, 'lock' : lock }# End of equalization_log if equalization_log['current_taxpayer_index'] == equalization_log['total_taxpayers']: equalization_log['end'] = True equalization_log['end_message'] = EQUALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/' else: equalization_log['end'] = False _Locals.log_eq_thread_logs_at_equalization_main_logs(equalization_execution_log=equalization_log,equalization_logger=equalization_logger,cron_logger=cron_logger) process_logger.info(2*LOG_INDENT + 'Updating taxpayer ... ') _Utilities.update_taxpayer_status(taxpayer,_Constants.EQUALIZATION,logger=process_logger) process_logger.info(2*LOG_INDENT + 'Equalized successfully. Logged at Equalization main logs') process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' + process_name.upper() + ' DONE SUCCESSFULLY \0/') process_logger.info(_Constants.LOG_SEPARATOR) return 'OK' except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: equalization_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def excute_initialization(taxpayers,shared_variables):# Taxpayers are the ones splitted for this specific subprocess try: # Shared variables: current_taxpayer = shared_variables['current_taxpayer'] total_taxpayers = shared_variables['total_taxpayers'] current_table_row = shared_variables['current_table_row'] forcing_execution = shared_variables['forcing_execution'] process_params = shared_variables['process_params'] forcing_period = False if 'year' in process_params: forcing_period = True lock = shared_variables['lock'] # Process: process_name = multiprocessing.current_process().name total_taxpayers_for_this_subprocess = len(taxpayers) # Process logger: process_logger = _Utilities.get_subprocess_logger(process_name,INITIALIZATION_LOGGING_CONFIG,logger=initialization_logger) process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(_Constants.LOG_SEPARATOR) process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' + process_name.upper()) taxpayers_initialized_counter = 0 process_logger.info(LOG_INDENT + 'Forcing execution: ' + str(forcing_execution)) process_logger.info(LOG_INDENT + 'Taxpayers: ' + str(total_taxpayers_for_this_subprocess)) for taxpayer in taxpayers: if not forcing_execution: _Utilities.update_current_taxpayer(_Constants.INITIALIZATION,taxpayer['identifier'],current_taxpayer.value+1,logger=process_logger) percentage_of_initialization_done = _Utilities.get_process_percentage_done(taxpayers_initialized_counter,total_taxpayers_for_this_subprocess) taxpayers_initialized_counter = taxpayers_initialized_counter + 1# Specific taxpayers (this thread's counter) process_logger.info(LOG_INDENT + '-> (' + str(taxpayers_initialized_counter) + '/' + str(total_taxpayers_for_this_subprocess) + ') ' + taxpayer['identifier'] + ' --- ' + percentage_of_initialization_done) initialization_execution_data = excute_initialization_for_taxpayer(forcing_period=forcing_period,forcing_execution=forcing_execution,taxpayer=taxpayer,process_logger=process_logger,process_params=process_params) with lock: current_taxpayer.value = current_taxpayer.value + 1 current_date = Datetime.now() initialization_execution_log = { 'date' : str(current_date)[:10], 'hour' : str(current_date)[10:-7], 'process_name' : process_name, 'current_taxpayer_index' : current_taxpayer.value, 'total_taxpayers' : total_taxpayers.value, 'identifier' : taxpayer['identifier'], 'new' : initialization_execution_data['new'], 'stored' : initialization_execution_data['stored'], 'year_initialized' : initialization_execution_data['year_initialized'], 'month_initialized' : initialization_execution_data['month_initialized'], 'percentage_initialized' : initialization_execution_data['percentage_initialized'], 'initialized' : initialization_execution_data['initialized'], # 'new' : 0, # 'stored' : 0, # 'year_initialized' : '2015', # 'month_initialized' : '02', # 'percentage_initialized' : 0, 'lock' : lock, 'current_table_row' : current_table_row }# End of initialization_execution_log if initialization_execution_log['current_taxpayer_index'] == initialization_execution_log['total_taxpayers']: initialization_execution_log['end'] = True initialization_execution_log['end_message'] = INITIALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/' else: initialization_execution_log['end'] = False _Locals.log_initiliazation_thread_logs_at_initialization_main_logs(initialization_execution_log=initialization_execution_log,initialization_logger=initialization_logger,cron_logger=cron_logger) if forcing_period or ('avoid_iteration' in initialization_execution_data and initialization_execution_data['avoid_iteration'] == True): process_logger.info(2*LOG_INDENT + 'NOT Updating initialization data for taxpayer ... ') else: process_logger.info(2*LOG_INDENT + 'Updating initialization data for taxpayer ... ') _Locals.update_initialization_data_for_taxpayer(taxpayer,initialization_execution_log,logger=process_logger) process_logger.info(2*LOG_INDENT + 'Synchronized successfully. Logged at SL1 main logs') process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' + process_name.upper() + ' DONE SUCCESSFULLY \0/') process_logger.info(_Constants.LOG_SEPARATOR) return 'OK' except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: initialization_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def excute_synchronization_layer_1( taxpayers, shared_variables ): # Taxpayers are the ones splitted for this specific subprocess try: # Shared variables: current_taxpayer = shared_variables['current_taxpayer'] total_taxpayers = shared_variables['total_taxpayers'] current_table_row = shared_variables['current_table_row'] forcing_execution = shared_variables['forcing_execution'] lock = shared_variables['lock'] # Process: process_name = multiprocessing.current_process().name total_taxpayers_for_this_subprocess = len(taxpayers) # Process logger: process_logger = _Utilities.get_subprocess_logger(process_name, SL1_LOGGING_CONFIG, logger=sl1_logger) # process_logger.info process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(' ') process_logger.info(_Constants.LOG_SEPARATOR) process_logger.info(SYNCHRONIZATION_LAYER_1_PROCESS_NAME + ' - ' + process_name.upper()) # Synchronization layer 1 data: sl1_data = _Locals.get_synchronization_layer_1_data( logger=process_logger) taxpayers_synchronized_counter = 0 process_logger.info(LOG_INDENT + 'Forcing execution: ' + str(forcing_execution)) process_logger.info(LOG_INDENT + 'Taxpayers: ' + str(total_taxpayers_for_this_subprocess)) for taxpayer in taxpayers: if not forcing_execution: _Utilities.update_current_taxpayer(_Constants.SL1, taxpayer['identifier'], current_taxpayer.value + 1, logger=process_logger) percentage_of_synchronization_done = _Utilities.get_process_percentage_done( taxpayers_synchronized_counter, total_taxpayers_for_this_subprocess) taxpayers_synchronized_counter = taxpayers_synchronized_counter + 1 # Specific taxpayers (this thread's counter) process_logger.info(LOG_INDENT + '-> (' + str(taxpayers_synchronized_counter) + '/' + str(total_taxpayers_for_this_subprocess) + ') ' + taxpayer['identifier'] + ' --- ' + percentage_of_synchronization_done) # ----- Improve: just if proccess goes down the taxpayers already synchronized (last 12 hours) are skipped last_sl1 = taxpayer['last_sl1'] today = Datetime.now() current_hour = today.hour hour = 0 if current_hour > 12: hour = 12 today = today.replace(hour=hour, minute=0, second=0) process_logger.info(2 * LOG_INDENT + 'Last SL1 done at: ' + str(last_sl1)) process_logger.info(2 * LOG_INDENT + 'Today date: ' + str(today)) process_logger.info(2 * LOG_INDENT + 'Comparing dates ... ') if last_sl1 is None: process_logger.info( 2 * LOG_INDENT + 'Sync is NEW, this taxpayer will continue ... ') elif last_sl1 >= today: process_logger.info( 2 * LOG_INDENT + 'Sync is up to date, this taxpayer will be skipped') continue else: process_logger.info( 2 * LOG_INDENT + 'Sync is NOT up to date, this taxpayer will continue ... ') # ----- sl1_execution_data = excute_synchronization_layer_1_for_taxpayer( forcing_execution=forcing_execution, taxpayer=taxpayer, sl1_data=sl1_data, process_logger=process_logger) with lock: current_taxpayer.value = current_taxpayer.value + 1 current_date = Datetime.now() sl1_execution_log = { 'date': str(current_date)[:10], 'hour': str(current_date)[10:-7], 'process_name': process_name, 'current_taxpayer_index': current_taxpayer.value, 'total_taxpayers': total_taxpayers.value, 'identifier': taxpayer['identifier'], 'new': sl1_execution_data['firmware']['new'], 'to_update': sl1_execution_data['forest_db']['before']['pending'], 'stored': sl1_execution_data['forest_db']['after']['new'], 'updated': sl1_execution_data['forest_db']['after']['updated'], 'completed': sl1_execution_data['forest_db']['after']['pending_completed'], 'pending': sl1_execution_data['forest_db']['after']['pending'], # 'new' : 0, # 'to_update' : 0, # 'stored' : 0, # 'updated' : 0, # 'completed' : 0, # 'pending' : 0, 'lock': lock, 'current_table_row': current_table_row } # End of sl1_execution_log if sl1_execution_log[ 'current_taxpayer_index'] == sl1_execution_log[ 'total_taxpayers']: sl1_execution_log['end'] = True sl1_execution_log[ 'end_message'] = SYNCHRONIZATION_LAYER_1_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/' else: sl1_execution_log['end'] = False _Locals.log_sl1_thread_logs_at_sl1_main_logs( sl1_execution_log=sl1_execution_log, sl1_logger=sl1_logger, cron_logger=cron_logger) if 'avoid_iteration' in sl1_execution_data and sl1_execution_data[ 'avoid_iteration'] == True: process_logger.info( 2 * LOG_INDENT + 'NOT Updating synchronization data for taxpayer ... ') else: process_logger.info( 2 * LOG_INDENT + 'Updating synchronization data for taxpayer ... ') _Locals.update_synchronization_data_for_taxpayer( taxpayer, sl1_execution_log, logger=process_logger) process_logger.info( 2 * LOG_INDENT + 'Synchronized successfully. Logged at SL1 main logs') process_logger.info(SYNCHRONIZATION_LAYER_1_PROCESS_NAME + ' - ' + process_name.upper() + ' DONE SUCCESSFULLY \0/') process_logger.info(_Constants.LOG_SEPARATOR) return 'OK' except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: sl1_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception