def equalize_dbs(
taxpayers, shared_variables
): # Taxpayers are the ones splitted for this specific subprocess
try:
# Shared variables:
current_taxpayer = shared_variables['current_taxpayer']
total_taxpayers = shared_variables['total_taxpayers']
current_table_row = shared_variables['current_table_row']
forcing_execution = shared_variables['forcing_execution']
process_params = shared_variables['process_params']
forcing_period = False
if 'year' in process_params or 'all' in process_params:
forcing_period = True
lock = shared_variables['lock']
# Process:
process_name = multiprocessing.current_process().name
total_taxpayers_for_this_subprocess = len(taxpayers)
# Process logger:
process_logger = _Utilities.get_subprocess_logger(
process_name,
EQUALIZATION_LOGGING_CONFIG,
logger=equalization_logger)
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(_Constants.LOG_SEPARATOR)
process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' +
process_name.upper())
taxpayers_equalized_counter = 0
process_logger.info(LOG_INDENT + 'Forcing execution: ' +
str(forcing_execution))
process_logger.info(LOG_INDENT + 'Taxpayers: ' +
str(total_taxpayers_for_this_subprocess))
for taxpayer in taxpayers:
if not forcing_execution:
_Utilities.update_current_taxpayer(_Constants.EQUALIZATION,
taxpayer['identifier'],
current_taxpayer.value + 1,
logger=process_logger)
percentage_of_equalization_done = _Utilities.get_process_percentage_done(
taxpayers_equalized_counter,
total_taxpayers_for_this_subprocess)
taxpayers_equalized_counter = taxpayers_equalized_counter + 1 # Specific taxpayers (this thread's counter)
process_logger.info(LOG_INDENT + '-> (' +
str(taxpayers_equalized_counter) + '/' +
str(total_taxpayers_for_this_subprocess) +
') ' + taxpayer['identifier'] + ' --- ' +
percentage_of_equalization_done)
equalization_data = equalize_dbs_for_a_taxpayer(
forcing_period=forcing_period,
forcing_execution=forcing_execution,
taxpayer=taxpayer,
process_logger=process_logger,
process_params=process_params)
with lock:
current_taxpayer.value = current_taxpayer.value + 1
current_date = Datetime.now()
equalization_log = {
'date':
str(current_date)[:10],
'hour':
str(current_date)[10:-7],
'process_name':
process_name,
'current_taxpayer_index':
current_taxpayer.value,
'total_taxpayers':
total_taxpayers.value,
'identifier':
taxpayer['identifier'],
'forest_db':
equalization_data['before']['forest_db'],
'corebook_db':
equalization_data['before']['corebook_db'],
'missing_in_forest_db':
equalization_data['before']['cb_but_not_in_f'],
'missing_in_cb_db':
equalization_data['before']['f_but_not_in_cb'],
'stored':
equalization_data['after']['stored'],
'errors':
equalization_data['after']['errors'],
# 'forest_db' : 90,
# 'corebook_db' : 10,
# 'missing_in_forest_db' : 10,
# 'missing_in_cb_db' : 11,
# 'stored' : 10,
# 'errors' : 1,
'current_table_row':
current_table_row,
'lock':
lock
} # End of equalization_log
if equalization_log['current_taxpayer_index'] == equalization_log[
'total_taxpayers']:
equalization_log['end'] = True
equalization_log[
'end_message'] = EQUALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/'
else:
equalization_log['end'] = False
_Locals.log_eq_thread_logs_at_equalization_main_logs(
equalization_execution_log=equalization_log,
equalization_logger=equalization_logger,
cron_logger=cron_logger)
process_logger.info(2 * LOG_INDENT + 'Updating taxpayer ... ')
_Utilities.update_taxpayer_status(taxpayer,
_Constants.EQUALIZATION,
logger=process_logger)
process_logger.info(
2 * LOG_INDENT +
'Equalized successfully. Logged at Equalization main logs')
process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' +
process_name.upper() + ' DONE SUCCESSFULLY \0/')
process_logger.info(_Constants.LOG_SEPARATOR)
return 'OK'
except Already_Handled_Exception as already_handled_exception:
raise already_handled_exception
except Exception as e:
equalization_logger.critical(e.message)
already_handled_exception = Already_Handled_Exception(e.message)
raise already_handled_exception
def excute_initialization(
taxpayers, shared_variables
): # Taxpayers are the ones splitted for this specific subprocess
try:
# Shared variables:
current_taxpayer = shared_variables['current_taxpayer']
total_taxpayers = shared_variables['total_taxpayers']
current_table_row = shared_variables['current_table_row']
forcing_execution = shared_variables['forcing_execution']
process_params = shared_variables['process_params']
forcing_period = False
if 'year' in process_params:
forcing_period = True
lock = shared_variables['lock']
# Process:
process_name = multiprocessing.current_process().name
total_taxpayers_for_this_subprocess = len(taxpayers)
# Process logger:
process_logger = _Utilities.get_subprocess_logger(
process_name,
INITIALIZATION_LOGGING_CONFIG,
logger=initialization_logger)
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(_Constants.LOG_SEPARATOR)
process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' +
process_name.upper())
taxpayers_initialized_counter = 0
process_logger.info(LOG_INDENT + 'Forcing execution: ' +
str(forcing_execution))
process_logger.info(LOG_INDENT + 'Taxpayers: ' +
str(total_taxpayers_for_this_subprocess))
for taxpayer in taxpayers:
if not forcing_execution:
_Utilities.update_current_taxpayer(_Constants.INITIALIZATION,
taxpayer['identifier'],
current_taxpayer.value + 1,
logger=process_logger)
percentage_of_initialization_done = _Utilities.get_process_percentage_done(
taxpayers_initialized_counter,
total_taxpayers_for_this_subprocess)
taxpayers_initialized_counter = taxpayers_initialized_counter + 1 # Specific taxpayers (this thread's counter)
process_logger.info(LOG_INDENT + '-> (' +
str(taxpayers_initialized_counter) + '/' +
str(total_taxpayers_for_this_subprocess) +
') ' + taxpayer['identifier'] + ' --- ' +
percentage_of_initialization_done)
initialization_execution_data = excute_initialization_for_taxpayer(
forcing_period=forcing_period,
forcing_execution=forcing_execution,
taxpayer=taxpayer,
process_logger=process_logger,
process_params=process_params)
with lock:
current_taxpayer.value = current_taxpayer.value + 1
current_date = Datetime.now()
initialization_execution_log = {
'date':
str(current_date)[:10],
'hour':
str(current_date)[10:-7],
'process_name':
process_name,
'current_taxpayer_index':
current_taxpayer.value,
'total_taxpayers':
total_taxpayers.value,
'identifier':
taxpayer['identifier'],
'new':
initialization_execution_data['new'],
'stored':
initialization_execution_data['stored'],
'year_initialized':
initialization_execution_data['year_initialized'],
'month_initialized':
initialization_execution_data['month_initialized'],
'percentage_initialized':
initialization_execution_data['percentage_initialized'],
'initialized':
initialization_execution_data['initialized'],
# 'new' : 0,
# 'stored' : 0,
# 'year_initialized' : '2015',
# 'month_initialized' : '02',
# 'percentage_initialized' : 0,
'lock':
lock,
'current_table_row':
current_table_row
} # End of initialization_execution_log
if initialization_execution_log[
'current_taxpayer_index'] == initialization_execution_log[
'total_taxpayers']:
initialization_execution_log['end'] = True
initialization_execution_log[
'end_message'] = INITIALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/'
else:
initialization_execution_log['end'] = False
_Locals.log_initiliazation_thread_logs_at_initialization_main_logs(
initialization_execution_log=initialization_execution_log,
initialization_logger=initialization_logger,
cron_logger=cron_logger)
if forcing_period or (
'avoid_iteration' in initialization_execution_data and
initialization_execution_data['avoid_iteration'] == True):
process_logger.info(
2 * LOG_INDENT +
'NOT Updating initialization data for taxpayer ... ')
else:
process_logger.info(
2 * LOG_INDENT +
'Updating initialization data for taxpayer ... ')
_Locals.update_initialization_data_for_taxpayer(
taxpayer,
initialization_execution_log,
logger=process_logger)
process_logger.info(
2 * LOG_INDENT +
'Synchronized successfully. Logged at SL1 main logs')
process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' +
process_name.upper() + ' DONE SUCCESSFULLY \0/')
process_logger.info(_Constants.LOG_SEPARATOR)
return 'OK'
except Already_Handled_Exception as already_handled_exception:
raise already_handled_exception
except Exception as e:
initialization_logger.critical(e.message)
already_handled_exception = Already_Handled_Exception(e.message)
raise already_handled_exception
def equalize_dbs(taxpayers,shared_variables):# Taxpayers are the ones splitted for this specific subprocess
try:
# Shared variables:
current_taxpayer = shared_variables['current_taxpayer']
total_taxpayers = shared_variables['total_taxpayers']
current_table_row = shared_variables['current_table_row']
forcing_execution = shared_variables['forcing_execution']
process_params = shared_variables['process_params']
forcing_period = False
if 'year' in process_params or 'all' in process_params:
forcing_period = True
lock = shared_variables['lock']
# Process:
process_name = multiprocessing.current_process().name
total_taxpayers_for_this_subprocess = len(taxpayers)
# Process logger:
process_logger = _Utilities.get_subprocess_logger(process_name,EQUALIZATION_LOGGING_CONFIG,logger=equalization_logger)
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(_Constants.LOG_SEPARATOR)
process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' + process_name.upper())
taxpayers_equalized_counter = 0
process_logger.info(LOG_INDENT + 'Forcing execution: ' + str(forcing_execution))
process_logger.info(LOG_INDENT + 'Taxpayers: ' + str(total_taxpayers_for_this_subprocess))
for taxpayer in taxpayers:
if not forcing_execution:
_Utilities.update_current_taxpayer(_Constants.EQUALIZATION,taxpayer['identifier'],current_taxpayer.value+1,logger=process_logger)
percentage_of_equalization_done = _Utilities.get_process_percentage_done(taxpayers_equalized_counter,total_taxpayers_for_this_subprocess)
taxpayers_equalized_counter = taxpayers_equalized_counter + 1# Specific taxpayers (this thread's counter)
process_logger.info(LOG_INDENT + '-> (' + str(taxpayers_equalized_counter) + '/' + str(total_taxpayers_for_this_subprocess) + ') ' + taxpayer['identifier'] + ' --- ' + percentage_of_equalization_done)
equalization_data = equalize_dbs_for_a_taxpayer(forcing_period=forcing_period,forcing_execution=forcing_execution,taxpayer=taxpayer,process_logger=process_logger,process_params=process_params)
with lock:
current_taxpayer.value = current_taxpayer.value + 1
current_date = Datetime.now()
equalization_log = {
'date' : str(current_date)[:10],
'hour' : str(current_date)[10:-7],
'process_name' : process_name,
'current_taxpayer_index' : current_taxpayer.value,
'total_taxpayers' : total_taxpayers.value,
'identifier' : taxpayer['identifier'],
'forest_db' : equalization_data['before']['forest_db'],
'corebook_db' : equalization_data['before']['corebook_db'],
'missing_in_forest_db' : equalization_data['before']['cb_but_not_in_f'],
'missing_in_cb_db' : equalization_data['before']['f_but_not_in_cb'],
'stored' : equalization_data['after']['stored'],
'errors' : equalization_data['after']['errors'],
# 'forest_db' : 90,
# 'corebook_db' : 10,
# 'missing_in_forest_db' : 10,
# 'missing_in_cb_db' : 11,
# 'stored' : 10,
# 'errors' : 1,
'current_table_row' : current_table_row,
'lock' : lock
}# End of equalization_log
if equalization_log['current_taxpayer_index'] == equalization_log['total_taxpayers']:
equalization_log['end'] = True
equalization_log['end_message'] = EQUALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/'
else:
equalization_log['end'] = False
_Locals.log_eq_thread_logs_at_equalization_main_logs(equalization_execution_log=equalization_log,equalization_logger=equalization_logger,cron_logger=cron_logger)
process_logger.info(2*LOG_INDENT + 'Updating taxpayer ... ')
_Utilities.update_taxpayer_status(taxpayer,_Constants.EQUALIZATION,logger=process_logger)
process_logger.info(2*LOG_INDENT + 'Equalized successfully. Logged at Equalization main logs')
process_logger.info(EQUALIZATION_PROCESS_NAME + ' - ' + process_name.upper() + ' DONE SUCCESSFULLY \0/')
process_logger.info(_Constants.LOG_SEPARATOR)
return 'OK'
except Already_Handled_Exception as already_handled_exception:
raise already_handled_exception
except Exception as e:
equalization_logger.critical(e.message)
already_handled_exception = Already_Handled_Exception(e.message)
raise already_handled_exception
def excute_initialization(taxpayers,shared_variables):# Taxpayers are the ones splitted for this specific subprocess
try:
# Shared variables:
current_taxpayer = shared_variables['current_taxpayer']
total_taxpayers = shared_variables['total_taxpayers']
current_table_row = shared_variables['current_table_row']
forcing_execution = shared_variables['forcing_execution']
process_params = shared_variables['process_params']
forcing_period = False
if 'year' in process_params:
forcing_period = True
lock = shared_variables['lock']
# Process:
process_name = multiprocessing.current_process().name
total_taxpayers_for_this_subprocess = len(taxpayers)
# Process logger:
process_logger = _Utilities.get_subprocess_logger(process_name,INITIALIZATION_LOGGING_CONFIG,logger=initialization_logger)
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(_Constants.LOG_SEPARATOR)
process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' + process_name.upper())
taxpayers_initialized_counter = 0
process_logger.info(LOG_INDENT + 'Forcing execution: ' + str(forcing_execution))
process_logger.info(LOG_INDENT + 'Taxpayers: ' + str(total_taxpayers_for_this_subprocess))
for taxpayer in taxpayers:
if not forcing_execution:
_Utilities.update_current_taxpayer(_Constants.INITIALIZATION,taxpayer['identifier'],current_taxpayer.value+1,logger=process_logger)
percentage_of_initialization_done = _Utilities.get_process_percentage_done(taxpayers_initialized_counter,total_taxpayers_for_this_subprocess)
taxpayers_initialized_counter = taxpayers_initialized_counter + 1# Specific taxpayers (this thread's counter)
process_logger.info(LOG_INDENT + '-> (' + str(taxpayers_initialized_counter) + '/' + str(total_taxpayers_for_this_subprocess) + ') ' + taxpayer['identifier'] + ' --- ' + percentage_of_initialization_done)
initialization_execution_data = excute_initialization_for_taxpayer(forcing_period=forcing_period,forcing_execution=forcing_execution,taxpayer=taxpayer,process_logger=process_logger,process_params=process_params)
with lock:
current_taxpayer.value = current_taxpayer.value + 1
current_date = Datetime.now()
initialization_execution_log = {
'date' : str(current_date)[:10],
'hour' : str(current_date)[10:-7],
'process_name' : process_name,
'current_taxpayer_index' : current_taxpayer.value,
'total_taxpayers' : total_taxpayers.value,
'identifier' : taxpayer['identifier'],
'new' : initialization_execution_data['new'],
'stored' : initialization_execution_data['stored'],
'year_initialized' : initialization_execution_data['year_initialized'],
'month_initialized' : initialization_execution_data['month_initialized'],
'percentage_initialized' : initialization_execution_data['percentage_initialized'],
'initialized' : initialization_execution_data['initialized'],
# 'new' : 0,
# 'stored' : 0,
# 'year_initialized' : '2015',
# 'month_initialized' : '02',
# 'percentage_initialized' : 0,
'lock' : lock,
'current_table_row' : current_table_row
}# End of initialization_execution_log
if initialization_execution_log['current_taxpayer_index'] == initialization_execution_log['total_taxpayers']:
initialization_execution_log['end'] = True
initialization_execution_log['end_message'] = INITIALIZATION_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/'
else:
initialization_execution_log['end'] = False
_Locals.log_initiliazation_thread_logs_at_initialization_main_logs(initialization_execution_log=initialization_execution_log,initialization_logger=initialization_logger,cron_logger=cron_logger)
if forcing_period or ('avoid_iteration' in initialization_execution_data and initialization_execution_data['avoid_iteration'] == True):
process_logger.info(2*LOG_INDENT + 'NOT Updating initialization data for taxpayer ... ')
else:
process_logger.info(2*LOG_INDENT + 'Updating initialization data for taxpayer ... ')
_Locals.update_initialization_data_for_taxpayer(taxpayer,initialization_execution_log,logger=process_logger)
process_logger.info(2*LOG_INDENT + 'Synchronized successfully. Logged at SL1 main logs')
process_logger.info(INITIALIZATION_PROCESS_NAME + ' - ' + process_name.upper() + ' DONE SUCCESSFULLY \0/')
process_logger.info(_Constants.LOG_SEPARATOR)
return 'OK'
except Already_Handled_Exception as already_handled_exception:
raise already_handled_exception
except Exception as e:
initialization_logger.critical(e.message)
already_handled_exception = Already_Handled_Exception(e.message)
raise already_handled_exception
def excute_synchronization_layer_1(
taxpayers, shared_variables
): # Taxpayers are the ones splitted for this specific subprocess
try:
# Shared variables:
current_taxpayer = shared_variables['current_taxpayer']
total_taxpayers = shared_variables['total_taxpayers']
current_table_row = shared_variables['current_table_row']
forcing_execution = shared_variables['forcing_execution']
lock = shared_variables['lock']
# Process:
process_name = multiprocessing.current_process().name
total_taxpayers_for_this_subprocess = len(taxpayers)
# Process logger:
process_logger = _Utilities.get_subprocess_logger(process_name,
SL1_LOGGING_CONFIG,
logger=sl1_logger)
# process_logger.info
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(' ')
process_logger.info(_Constants.LOG_SEPARATOR)
process_logger.info(SYNCHRONIZATION_LAYER_1_PROCESS_NAME + ' - ' +
process_name.upper())
# Synchronization layer 1 data:
sl1_data = _Locals.get_synchronization_layer_1_data(
logger=process_logger)
taxpayers_synchronized_counter = 0
process_logger.info(LOG_INDENT + 'Forcing execution: ' +
str(forcing_execution))
process_logger.info(LOG_INDENT + 'Taxpayers: ' +
str(total_taxpayers_for_this_subprocess))
for taxpayer in taxpayers:
if not forcing_execution:
_Utilities.update_current_taxpayer(_Constants.SL1,
taxpayer['identifier'],
current_taxpayer.value + 1,
logger=process_logger)
percentage_of_synchronization_done = _Utilities.get_process_percentage_done(
taxpayers_synchronized_counter,
total_taxpayers_for_this_subprocess)
taxpayers_synchronized_counter = taxpayers_synchronized_counter + 1 # Specific taxpayers (this thread's counter)
process_logger.info(LOG_INDENT + '-> (' +
str(taxpayers_synchronized_counter) + '/' +
str(total_taxpayers_for_this_subprocess) +
') ' + taxpayer['identifier'] + ' --- ' +
percentage_of_synchronization_done)
# ----- Improve: just if proccess goes down the taxpayers already synchronized (last 12 hours) are skipped
last_sl1 = taxpayer['last_sl1']
today = Datetime.now()
current_hour = today.hour
hour = 0
if current_hour > 12:
hour = 12
today = today.replace(hour=hour, minute=0, second=0)
process_logger.info(2 * LOG_INDENT + 'Last SL1 done at: ' +
str(last_sl1))
process_logger.info(2 * LOG_INDENT + 'Today date: ' +
str(today))
process_logger.info(2 * LOG_INDENT + 'Comparing dates ... ')
if last_sl1 is None:
process_logger.info(
2 * LOG_INDENT +
'Sync is NEW, this taxpayer will continue ... ')
elif last_sl1 >= today:
process_logger.info(
2 * LOG_INDENT +
'Sync is up to date, this taxpayer will be skipped')
continue
else:
process_logger.info(
2 * LOG_INDENT +
'Sync is NOT up to date, this taxpayer will continue ... ')
# -----
sl1_execution_data = excute_synchronization_layer_1_for_taxpayer(
forcing_execution=forcing_execution,
taxpayer=taxpayer,
sl1_data=sl1_data,
process_logger=process_logger)
with lock:
current_taxpayer.value = current_taxpayer.value + 1
current_date = Datetime.now()
sl1_execution_log = {
'date':
str(current_date)[:10],
'hour':
str(current_date)[10:-7],
'process_name':
process_name,
'current_taxpayer_index':
current_taxpayer.value,
'total_taxpayers':
total_taxpayers.value,
'identifier':
taxpayer['identifier'],
'new':
sl1_execution_data['firmware']['new'],
'to_update':
sl1_execution_data['forest_db']['before']['pending'],
'stored':
sl1_execution_data['forest_db']['after']['new'],
'updated':
sl1_execution_data['forest_db']['after']['updated'],
'completed':
sl1_execution_data['forest_db']['after']['pending_completed'],
'pending':
sl1_execution_data['forest_db']['after']['pending'],
# 'new' : 0,
# 'to_update' : 0,
# 'stored' : 0,
# 'updated' : 0,
# 'completed' : 0,
# 'pending' : 0,
'lock':
lock,
'current_table_row':
current_table_row
} # End of sl1_execution_log
if sl1_execution_log[
'current_taxpayer_index'] == sl1_execution_log[
'total_taxpayers']:
sl1_execution_log['end'] = True
sl1_execution_log[
'end_message'] = SYNCHRONIZATION_LAYER_1_PROCESS_NAME + ' DONE SUCCESSFULLY \\0/'
else:
sl1_execution_log['end'] = False
_Locals.log_sl1_thread_logs_at_sl1_main_logs(
sl1_execution_log=sl1_execution_log,
sl1_logger=sl1_logger,
cron_logger=cron_logger)
if 'avoid_iteration' in sl1_execution_data and sl1_execution_data[
'avoid_iteration'] == True:
process_logger.info(
2 * LOG_INDENT +
'NOT Updating synchronization data for taxpayer ... ')
else:
process_logger.info(
2 * LOG_INDENT +
'Updating synchronization data for taxpayer ... ')
_Locals.update_synchronization_data_for_taxpayer(
taxpayer, sl1_execution_log, logger=process_logger)
process_logger.info(
2 * LOG_INDENT +
'Synchronized successfully. Logged at SL1 main logs')
process_logger.info(SYNCHRONIZATION_LAYER_1_PROCESS_NAME + ' - ' +
process_name.upper() + ' DONE SUCCESSFULLY \0/')
process_logger.info(_Constants.LOG_SEPARATOR)
return 'OK'
except Already_Handled_Exception as already_handled_exception:
raise already_handled_exception
except Exception as e:
sl1_logger.critical(e.message)
already_handled_exception = Already_Handled_Exception(e.message)
raise already_handled_exception