def validate(process_name,mode=None,server_index=None,debug_execution=False): try: validated = True # Forest-Croning functione existance: cron_logger.info(2*LOG_INDENT + 'Checking Forest-Cron function existance ... ') if not process_name in PROCESS_HANDLER_CONFIG: cron_logger.info(LOG_INDENT + 'Process ' + process_name + ' does not exists at Forest-Cron') return False else: process_instance = PROCESS_HANDLER_CONFIG[process_name] if process_instance is None: cron_logger.info(LOG_INDENT + 'Invalid process name ' + process_name) return False # Process availability: if debug_execution is True: return validated if mode == _Constants.BALANCER_MODE: cron_logger.info(2*LOG_INDENT + 'Balancer mode, checking processs ' + process_name + ' availability ... ') process_availability = _Utilities.check_process_availability(process_name,logger=cron_logger) if process_availability is not True: cron_logger.info(LOG_INDENT + 'Process ' + process_name + ' is not available') return False return True elif mode == _Constants.SERVER_MODE: cron_logger.info(2*LOG_INDENT + 'Server mode, checking processs ' + process_name + ' availability ... ') process_availability = _Utilities.check_process_availability(process_name,logger=cron_logger) if process_availability is False: cron_logger.info(2*LOG_INDENT + 'Checking server ' + str(server_index) + ' of process ' + process_name + ' availability ... ') server_availability = _Utilities.check_process_server_availability(process_name,server_index,logger=cron_logger) if server_availability is not True: cron_logger.info(LOG_INDENT + 'Server ' + str(server_index) + ' of process ' + process_name + ' is not available') return False return True cron_logger.info(2*LOG_INDENT + 'Process ' + process_name + ' is available (then server processes are not available) ... ') return False except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: cron_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def validate(process_name, mode=None, server_index=None, debug_execution=False): try: validated = True # Forest-Croning functione existance: cron_logger.info(2 * LOG_INDENT + 'Checking Forest-Cron function existance ... ') if not process_name in PROCESS_HANDLER_CONFIG: cron_logger.info(LOG_INDENT + 'Process ' + process_name + ' does not exists at Forest-Cron') return False else: process_instance = PROCESS_HANDLER_CONFIG[process_name] if process_instance is None: cron_logger.info(LOG_INDENT + 'Invalid process name ' + process_name) return False # Process availability: if debug_execution is True: return validated if mode == _Constants.BALANCER_MODE: cron_logger.info(2 * LOG_INDENT + 'Balancer mode, checking processs ' + process_name + ' availability ... ') process_availability = _Utilities.check_process_availability( process_name, logger=cron_logger) if process_availability is not True: cron_logger.info(LOG_INDENT + 'Process ' + process_name + ' is not available') return False return True elif mode == _Constants.SERVER_MODE: cron_logger.info(2 * LOG_INDENT + 'Server mode, checking processs ' + process_name + ' availability ... ') process_availability = _Utilities.check_process_availability( process_name, logger=cron_logger) if process_availability is False: cron_logger.info(2 * LOG_INDENT + 'Checking server ' + str(server_index) + ' of process ' + process_name + ' availability ... ') server_availability = _Utilities.check_process_server_availability( process_name, server_index, logger=cron_logger) if server_availability is not True: cron_logger.info(LOG_INDENT + 'Server ' + str(server_index) + ' of process ' + process_name + ' is not available') return False return True cron_logger.info( 2 * LOG_INDENT + 'Process ' + process_name + ' is available (then server processes are not available) ... ') return False except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: cron_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def execute(process): try: start_time = time.time() process_name = process['name'] process_params = process['params'] forcing_execution = False forcing_identifiers = None forest_mode = PROCESS_HANDLER_CONFIG['forest_mode'] server_index = PROCESS_HANDLER_CONFIG[ 'server_index'] # The index related to this server (in case it is configured in SERVER mode) if process_params is not None and 'identifiers' in process_params and process_params[ 'identifiers'] is not None: forcing_execution = True forest_mode = _Constants.SERVER_MODE # Forcing is executed as server mode forcing_identifiers = process_params['identifiers'] if _Utilities.validate_forcing_identifiers( forcing_identifiers, process_name, logger=cron_logger) is False: print 'Identifiers are not valid, this means some identfiers do not exist, or, if they exist, their status is not equal to ' + process_name sys.exit() else: print 'Identifiers are OK' if (process_name == _Constants.INITIALIZATION or process_name == _Constants.EQUALIZATION) and forcing_execution: if _Utilities.validate_params(process_params, logger=cron_logger) is False: print 'Period params are wrong' sys.exit() debug_execution = process['debug'] if forcing_execution is False: cron_logger.info(LOG_INDENT + 'Validating CRON Process') process_availability = _Utilities.check_process_availability( 'cron', debug_execution=debug_execution, logger=cron_logger) if process_availability is not True: cron_logger.info(LOG_INDENT + 'CRON is suspended') suspended_at = Datetime.now() _Utilities.update_cron_process_log( 'cron', logger=cron_logger, suspended_at=suspended_at, debug_execution=debug_execution) return False cron_logger.info(LOG_INDENT + 'Validating process ... ') process_is_valid = validate(process_name, forest_mode, server_index, debug_execution=debug_execution) else: process_is_valid = True print 'Forcing execution of ' + process_name cron_logger.info(LOG_INDENT + 'Forcing execution') if process_is_valid: cron_logger.info(LOG_INDENT + 'Updating cron db log ... ') if forcing_execution is False: _Utilities.update_cron_process_log( process_name, logger=cron_logger, debug_execution=debug_execution) cron_logger.info(LOG_INDENT + 'Getting instance of ' + process_name) # Process data: SPECIFIC_PROCESS_CONFIG_DATA = PROCESS_HANDLER_CONFIG[process_name] specific_process_logger = SPECIFIC_PROCESS_CONFIG_DATA[ 'specific_process_logger'] process_instance = SPECIFIC_PROCESS_CONFIG_DATA['process_instance'] default_log = SPECIFIC_PROCESS_CONFIG_DATA['default_log'] cron_logger_starting_message = SPECIFIC_PROCESS_CONFIG_DATA[ 'cron_logger_starting_message'] process_name = SPECIFIC_PROCESS_CONFIG_DATA['process_name'] process_file_name = SPECIFIC_PROCESS_CONFIG_DATA[ 'process_file_name'] threads = SPECIFIC_PROCESS_CONFIG_DATA['threads'] cron_logger.info(LOG_INDENT + 'Running in mode: ' + str(forest_mode)) specific_shared_variables = SPECIFIC_PROCESS_CONFIG_DATA[ 'specific_shared_variables'] # Get process at db: cron_logger.info(LOG_INDENT + 'Getting process ' + process_name + ' at db') process = _Utilities.get_db_process(process_name) from_taxpayer = None if forcing_identifiers is not None: print 'Process will be forced for ' + str( len(forcing_identifiers)) cron_logger.info(LOG_INDENT + 'This process will be force for ' + str(len(forcing_identifiers))) elif debug_execution is True: cron_logger.info(LOG_INDENT + 'This process will run in debugging mode') elif 'current_taxpayer' in process: from_taxpayer = process[ 'current_taxpayer'] # If process fails or if it is stopped it will start from this taxpayer cron_logger.info(LOG_INDENT + 'This process will run from taxpayer ' + from_taxpayer) else: cron_logger.info(LOG_INDENT + 'This process will run for all taxpayers') # Update default log default_log = _Utilities.add_defalut_data_to_default_log( default_log) # Logging: if debug_execution is not True: cron_logger.info(LOG_INDENT + 'Logging calling at cron procesess ... ') _Utilities.log_at_cron_processes(process) cron_logger.info(LOG_INDENT + 'Getting taxpayers for this process ... ') taxpayers = _Utilities.get_taxpayers_for_a_specific_process( process_name, forcing_identifiers=forcing_identifiers, limit=None, from_taxpayer=from_taxpayer, logger=cron_logger, debug_execution=debug_execution, server_index=server_index, mode=forest_mode) # Set unavailable: if forest_mode == _Constants.SERVER_MODE and forcing_execution is False: cron_logger.info(2 * LOG_INDENT + 'Setting server ' + str(server_index) + ' unavailable for ' + process_name) _Utilities.set_process_server_unavailable(process_name, server_index, logger=cron_logger) elif forest_mode == _Constants.BALANCER_MODE and forcing_execution is False: cron_logger.info(LOG_INDENT + 'Setting process ' + process_name + ' unavailable and could server start') process_availability = _Utilities.set_process_unavailable( process_name, taxpayers=taxpayers, logger=cron_logger, debug_execution=debug_execution, threads=threads) cron_logger.info(LOG_INDENT + process_name + ' availability: ' + str(process_availability)) # Multi-threading execution: cron_logger.info(LOG_INDENT + 'Executing ... ') cron_logger.info(2 * LOG_INDENT + 'Process name: ' + process_name) if forest_mode == _Constants.BALANCER_MODE: cron_logger.info(2 * LOG_INDENT + 'Servers: ' + str(threads)) elif forest_mode == _Constants.SERVER_MODE: cron_logger.info(2 * LOG_INDENT + 'Threads: ' + str(threads)) else: cron_logger.info(2 * LOG_INDENT + 'UNAVAILABLE MODE') return cron_logger.info(2 * LOG_INDENT + 'Taxpayers: ' + str(len(taxpayers))) cron_logger.info(2 * LOG_INDENT + 'Params: ' + str(process_params)) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Performing ' + process_name + ' ... If you wish to check progress go to the log files' execute_with_multiprocessing( process_file_name=process_file_name, specific_process_logger=specific_process_logger, default_log=default_log, cron_logger_starting_message=cron_logger_starting_message, process_name=process_name, process_instance=process_instance, threads=threads, specific_shared_variables=specific_shared_variables, taxpayers=taxpayers, mode=forest_mode, forcing_execution=forcing_execution, process_params=process_params) end_time = time.time() process_duration = (end_time - start_time) / 3600 # in hours log_process_duration = False if len(taxpayers) > 0: log_process_duration = True if forest_mode == _Constants.SERVER_MODE and not forcing_execution: cron_logger.info(2 * LOG_INDENT + 'Setting server ' + str(server_index) + ' available for ' + process_name) _Utilities.set_process_server_available(process_name, server_index, logger=cron_logger) # BALANCER MODE is set available once all servers are available: all_servers_are_available = _Utilities.check_process_servers_availability( process_name, logger=cron_logger) cron_logger.info(2 * LOG_INDENT + 'All servers availability ' + str(all_servers_are_available)) if all_servers_are_available: cron_logger.info(2 * LOG_INDENT + 'Setting process available for ' + process_name) if process_name == _Constants.SL1: message = HAPPY_FACE + 'Ya acabe de sincronizar a todos los contribuyentes carnal y va de nuez' _Utilities.send_message_to_forest_telegram_contacts( message) _Utilities.set_process_available( process_name, process_duration=process_duration, logger=cron_logger, log_process_duration=log_process_duration, debug_execution=debug_execution) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Execution forced successfully' print 'You are awesome! :)' return if process_name == _Constants.EQUALIZATION: cron_logger.info(2 * LOG_INDENT + 'Setting process available for ' + process_name) _Utilities.set_process_available( process_name, process_duration=process_duration, logger=cron_logger, log_process_duration=log_process_duration, debug_execution=debug_execution) else: cron_logger.info(LOG_INDENT + 'End of execution') except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: cron_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def execute(process): try: start_time = time.time() process_name = process['name'] process_params = process['params'] forcing_execution = False forcing_identifiers = None forest_mode = PROCESS_HANDLER_CONFIG['forest_mode'] server_index = PROCESS_HANDLER_CONFIG['server_index']# The index related to this server (in case it is configured in SERVER mode) if process_params is not None and 'identifiers' in process_params and process_params['identifiers'] is not None: forcing_execution = True forest_mode = _Constants.SERVER_MODE# Forcing is executed as server mode forcing_identifiers = process_params['identifiers'] if _Utilities.validate_forcing_identifiers(forcing_identifiers,process_name,logger=cron_logger) is False: print 'Identifiers are not valid, this means some identfiers do not exist, or, if they exist, their status is not equal to ' + process_name sys.exit() else: print 'Identifiers are OK' if (process_name == _Constants.INITIALIZATION or process_name == _Constants.EQUALIZATION) and forcing_execution: if _Utilities.validate_params(process_params,logger=cron_logger) is False: print 'Period params are wrong' sys.exit() debug_execution = process['debug'] if forcing_execution is False: cron_logger.info(LOG_INDENT + 'Validating CRON Process') process_availability = _Utilities.check_process_availability('cron',debug_execution=debug_execution,logger=cron_logger) if process_availability is not True: cron_logger.info(LOG_INDENT + 'CRON is suspended') suspended_at = Datetime.now() _Utilities.update_cron_process_log('cron',logger=cron_logger,suspended_at=suspended_at,debug_execution=debug_execution) return False cron_logger.info(LOG_INDENT + 'Validating process ... ') process_is_valid = validate(process_name,forest_mode,server_index,debug_execution=debug_execution) else: process_is_valid = True print 'Forcing execution of ' + process_name cron_logger.info(LOG_INDENT + 'Forcing execution') if process_is_valid: cron_logger.info(LOG_INDENT + 'Updating cron db log ... ') if forcing_execution is False: _Utilities.update_cron_process_log(process_name,logger=cron_logger,debug_execution=debug_execution) cron_logger.info(LOG_INDENT + 'Getting instance of ' + process_name) # Process data: SPECIFIC_PROCESS_CONFIG_DATA = PROCESS_HANDLER_CONFIG[process_name] specific_process_logger = SPECIFIC_PROCESS_CONFIG_DATA['specific_process_logger'] process_instance = SPECIFIC_PROCESS_CONFIG_DATA['process_instance'] default_log = SPECIFIC_PROCESS_CONFIG_DATA['default_log'] cron_logger_starting_message = SPECIFIC_PROCESS_CONFIG_DATA['cron_logger_starting_message'] process_name = SPECIFIC_PROCESS_CONFIG_DATA['process_name'] process_file_name = SPECIFIC_PROCESS_CONFIG_DATA['process_file_name'] threads = SPECIFIC_PROCESS_CONFIG_DATA['threads'] cron_logger.info(LOG_INDENT + 'Running in mode: ' + str(forest_mode)) specific_shared_variables = SPECIFIC_PROCESS_CONFIG_DATA['specific_shared_variables'] # Get process at db: cron_logger.info(LOG_INDENT + 'Getting process ' + process_name + ' at db') process = _Utilities.get_db_process(process_name) from_taxpayer = None if forcing_identifiers is not None: print 'Process will be forced for ' + str(len(forcing_identifiers)) cron_logger.info(LOG_INDENT + 'This process will be force for ' + str(len(forcing_identifiers))) elif debug_execution is True: cron_logger.info(LOG_INDENT + 'This process will run in debugging mode') elif 'current_taxpayer' in process: from_taxpayer = process['current_taxpayer']# If process fails or if it is stopped it will start from this taxpayer cron_logger.info(LOG_INDENT + 'This process will run from taxpayer ' + from_taxpayer) else: cron_logger.info(LOG_INDENT + 'This process will run for all taxpayers') # Update default log default_log = _Utilities.add_defalut_data_to_default_log(default_log) # Logging: if debug_execution is not True: cron_logger.info(LOG_INDENT + 'Logging calling at cron procesess ... ') _Utilities.log_at_cron_processes(process) cron_logger.info(LOG_INDENT + 'Getting taxpayers for this process ... ') taxpayers = _Utilities.get_taxpayers_for_a_specific_process(process_name,forcing_identifiers=forcing_identifiers,limit=None,from_taxpayer=from_taxpayer,logger=cron_logger,debug_execution=debug_execution,server_index=server_index,mode=forest_mode) # Set unavailable: if forest_mode == _Constants.SERVER_MODE and forcing_execution is False: cron_logger.info(2*LOG_INDENT + 'Setting server ' + str(server_index) + ' unavailable for ' + process_name) _Utilities.set_process_server_unavailable(process_name,server_index,logger=cron_logger) elif forest_mode == _Constants.BALANCER_MODE and forcing_execution is False: cron_logger.info(LOG_INDENT + 'Setting process ' + process_name + ' unavailable and could server start') process_availability = _Utilities.set_process_unavailable(process_name,taxpayers=taxpayers,logger=cron_logger,debug_execution=debug_execution,threads=threads) cron_logger.info(LOG_INDENT + process_name + ' availability: ' + str(process_availability)) # Multi-threading execution: cron_logger.info(LOG_INDENT + 'Executing ... ') cron_logger.info(2*LOG_INDENT + 'Process name: ' + process_name) if forest_mode == _Constants.BALANCER_MODE: cron_logger.info(2*LOG_INDENT + 'Servers: ' + str(threads)) elif forest_mode == _Constants.SERVER_MODE: cron_logger.info(2*LOG_INDENT + 'Threads: ' + str(threads)) else: cron_logger.info(2*LOG_INDENT + 'UNAVAILABLE MODE') return cron_logger.info(2*LOG_INDENT + 'Taxpayers: ' + str(len(taxpayers))) cron_logger.info(2*LOG_INDENT + 'Params: ' + str(process_params)) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Performing ' + process_name + ' ... If you wish to check progress go to the log files' execute_with_multiprocessing(process_file_name=process_file_name,specific_process_logger=specific_process_logger,default_log=default_log,cron_logger_starting_message=cron_logger_starting_message,process_name=process_name,process_instance=process_instance,threads=threads,specific_shared_variables=specific_shared_variables,taxpayers=taxpayers,mode=forest_mode,forcing_execution=forcing_execution,process_params=process_params) end_time = time.time() process_duration = (end_time - start_time)/3600# in hours log_process_duration = False if len(taxpayers) > 0: log_process_duration = True if forest_mode == _Constants.SERVER_MODE and not forcing_execution: cron_logger.info(2*LOG_INDENT + 'Setting server ' + str(server_index) + ' available for ' + process_name) _Utilities.set_process_server_available(process_name,server_index,logger=cron_logger) # BALANCER MODE is set available once all servers are available: all_servers_are_available = _Utilities.check_process_servers_availability(process_name,logger=cron_logger) cron_logger.info(2*LOG_INDENT + 'All servers availability ' + str(all_servers_are_available)) if all_servers_are_available: cron_logger.info(2*LOG_INDENT + 'Setting process available for ' + process_name) if process_name == _Constants.SL1: message = HAPPY_FACE + 'Ya acabe de sincronizar a todos los contribuyentes carnal y va de nuez' _Utilities.send_message_to_forest_telegram_contacts(message) _Utilities.set_process_available(process_name,process_duration=process_duration,logger=cron_logger,log_process_duration=log_process_duration,debug_execution=debug_execution) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Execution forced successfully' print 'You are awesome! :)' return if process_name == _Constants.EQUALIZATION: cron_logger.info(2*LOG_INDENT + 'Setting process available for ' + process_name) _Utilities.set_process_available(process_name,process_duration=process_duration,logger=cron_logger,log_process_duration=log_process_duration,debug_execution=debug_execution) else: cron_logger.info(LOG_INDENT + 'End of execution') except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: cron_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception