Beispiel #1
0
def main(argv):
	hey_execution = get_command_line_params(argv)
	cron_logger = _Utilities.setup_logger('cron')
	current_hour = Datetime.now().hour
	chat_id = None
	if hey_execution:
		hey = _Utilities.hey(logger=cron_logger,server_index=SERVER_INDEX)
		if hey is False:
			return
		chat_id = hey
	cron_logger.info('------------------------------------------------------------------------')
	cron_logger.info(LOG_INDENT + 'Performing Forest Supervision')
	processes = get_processes_in_this_mode()
	for process in processes:
		if process == _Constants.INITIALIZATION:
			taxpayers = _Utilities.get_taxpayers_for_a_specific_process(process)
			taxpayers_in_init = len(taxpayers)
			if taxpayers_in_init > 0:
				last_log_date = get_last_log_time_of_a_process(process)
			else:
				last_log_date = True
		else:
			last_log_date = get_last_log_time_of_a_process(process)		
		everything_ok = is_everything_ok(last_log_date)	
		cron_logger.info(LOG_INDENT + process.upper())
		current_date = Datetime.now()
		year = current_date.year
		month = current_date.month
		day = current_date.day
		begin_of_the_day = Datetime(year,month,day)		
		if last_log_date > begin_of_the_day:
			last_log_date_hour = str(last_log_date)[11:11+8]
		else:
			last_log_date_hour = str(last_log_date)
		if everything_ok:
			message = HAPPY_FACE + 'Todo chido con ' + process + ' en Forest_' + str(SERVER_INDEX) + '. Tengo el ultimo registro a las ' + str(last_log_date_hour)
			cron_logger.info(2*LOG_INDENT + 'Everyting ok. Last log was at: ' + str(last_log_date_hour))
			if chat_id is not None:
				_Utilities.send_message_to_forest_telegram_contacts(message,chat_ids=[chat_id])
			else:
				_Utilities.send_message_to_forest_telegram_contacts(message)
		else:
			message = 'Tengo un ' + 3*telegram.Emoji.PILE_OF_POO + 1*CHANGITO + ' en Forest_' + str(SERVER_INDEX) + '. El ultimo registro a las ' + str(last_log_date_hour)
			cron_logger.info(2*LOG_INDENT + 'There is a PROBLEM. Last log was at: ' + str(last_log_date_hour))
			if chat_id is not None:
				_Utilities.send_message_to_forest_telegram_contacts(message,chat_ids=[chat_id])
			else:
				_Utilities.send_message_to_forest_telegram_contacts(message)
Beispiel #2
0
def execute(process):
    try:
        start_time = time.time()
        process_name = process['name']
        process_params = process['params']
        forcing_execution = False
        forcing_identifiers = None
        forest_mode = PROCESS_HANDLER_CONFIG['forest_mode']
        server_index = PROCESS_HANDLER_CONFIG[
            'server_index']  # The index related to this server (in case it is configured in SERVER mode)
        if process_params is not None and 'identifiers' in process_params and process_params[
                'identifiers'] is not None:
            forcing_execution = True
            forest_mode = _Constants.SERVER_MODE  # Forcing is executed as server mode
            forcing_identifiers = process_params['identifiers']
            if _Utilities.validate_forcing_identifiers(
                    forcing_identifiers, process_name,
                    logger=cron_logger) is False:
                print 'Identifiers are not valid, this means some identfiers do not exist, or, if they exist, their status is not equal to ' + process_name
                sys.exit()
            else:
                print 'Identifiers are OK'
        if (process_name == _Constants.INITIALIZATION or process_name
                == _Constants.EQUALIZATION) and forcing_execution:
            if _Utilities.validate_params(process_params,
                                          logger=cron_logger) is False:
                print 'Period params are wrong'
                sys.exit()
        debug_execution = process['debug']
        if forcing_execution is False:
            cron_logger.info(LOG_INDENT + 'Validating CRON Process')
            process_availability = _Utilities.check_process_availability(
                'cron', debug_execution=debug_execution, logger=cron_logger)
            if process_availability is not True:
                cron_logger.info(LOG_INDENT + 'CRON is suspended')
                suspended_at = Datetime.now()
                _Utilities.update_cron_process_log(
                    'cron',
                    logger=cron_logger,
                    suspended_at=suspended_at,
                    debug_execution=debug_execution)
                return False
            cron_logger.info(LOG_INDENT + 'Validating process ... ')
            process_is_valid = validate(process_name,
                                        forest_mode,
                                        server_index,
                                        debug_execution=debug_execution)
        else:
            process_is_valid = True
            print 'Forcing execution of ' + process_name
            cron_logger.info(LOG_INDENT + 'Forcing execution')
        if process_is_valid:
            cron_logger.info(LOG_INDENT + 'Updating cron db log ... ')
            if forcing_execution is False:
                _Utilities.update_cron_process_log(
                    process_name,
                    logger=cron_logger,
                    debug_execution=debug_execution)
            cron_logger.info(LOG_INDENT + 'Getting instance of ' +
                             process_name)
            # Process data:
            SPECIFIC_PROCESS_CONFIG_DATA = PROCESS_HANDLER_CONFIG[process_name]
            specific_process_logger = SPECIFIC_PROCESS_CONFIG_DATA[
                'specific_process_logger']
            process_instance = SPECIFIC_PROCESS_CONFIG_DATA['process_instance']
            default_log = SPECIFIC_PROCESS_CONFIG_DATA['default_log']
            cron_logger_starting_message = SPECIFIC_PROCESS_CONFIG_DATA[
                'cron_logger_starting_message']
            process_name = SPECIFIC_PROCESS_CONFIG_DATA['process_name']
            process_file_name = SPECIFIC_PROCESS_CONFIG_DATA[
                'process_file_name']
            threads = SPECIFIC_PROCESS_CONFIG_DATA['threads']
            cron_logger.info(LOG_INDENT + 'Running in mode: ' +
                             str(forest_mode))
            specific_shared_variables = SPECIFIC_PROCESS_CONFIG_DATA[
                'specific_shared_variables']
            # Get process at db:
            cron_logger.info(LOG_INDENT + 'Getting process ' + process_name +
                             ' at db')
            process = _Utilities.get_db_process(process_name)
            from_taxpayer = None
            if forcing_identifiers is not None:
                print 'Process will be forced for ' + str(
                    len(forcing_identifiers))
                cron_logger.info(LOG_INDENT +
                                 'This process will be force for ' +
                                 str(len(forcing_identifiers)))
            elif debug_execution is True:
                cron_logger.info(LOG_INDENT +
                                 'This process will run in debugging mode')
            elif 'current_taxpayer' in process:
                from_taxpayer = process[
                    'current_taxpayer']  # If process fails or if it is stopped it will start from this taxpayer
                cron_logger.info(LOG_INDENT +
                                 'This process will run from taxpayer ' +
                                 from_taxpayer)
            else:
                cron_logger.info(LOG_INDENT +
                                 'This process will run for all taxpayers')
            # Update default log
            default_log = _Utilities.add_defalut_data_to_default_log(
                default_log)
            # Logging:
            if debug_execution is not True:
                cron_logger.info(LOG_INDENT +
                                 'Logging calling at cron procesess ... ')
                _Utilities.log_at_cron_processes(process)
            cron_logger.info(LOG_INDENT +
                             'Getting taxpayers for this process ... ')
            taxpayers = _Utilities.get_taxpayers_for_a_specific_process(
                process_name,
                forcing_identifiers=forcing_identifiers,
                limit=None,
                from_taxpayer=from_taxpayer,
                logger=cron_logger,
                debug_execution=debug_execution,
                server_index=server_index,
                mode=forest_mode)
            # Set unavailable:
            if forest_mode == _Constants.SERVER_MODE and forcing_execution is False:
                cron_logger.info(2 * LOG_INDENT + 'Setting server ' +
                                 str(server_index) + ' unavailable for ' +
                                 process_name)
                _Utilities.set_process_server_unavailable(process_name,
                                                          server_index,
                                                          logger=cron_logger)
            elif forest_mode == _Constants.BALANCER_MODE and forcing_execution is False:
                cron_logger.info(LOG_INDENT + 'Setting process ' +
                                 process_name +
                                 ' unavailable and could server start')
                process_availability = _Utilities.set_process_unavailable(
                    process_name,
                    taxpayers=taxpayers,
                    logger=cron_logger,
                    debug_execution=debug_execution,
                    threads=threads)
                cron_logger.info(LOG_INDENT + process_name +
                                 ' availability: ' + str(process_availability))
            # Multi-threading execution:
            cron_logger.info(LOG_INDENT + 'Executing ... ')
            cron_logger.info(2 * LOG_INDENT + 'Process name: ' + process_name)
            if forest_mode == _Constants.BALANCER_MODE:
                cron_logger.info(2 * LOG_INDENT + 'Servers:      ' +
                                 str(threads))
            elif forest_mode == _Constants.SERVER_MODE:
                cron_logger.info(2 * LOG_INDENT + 'Threads:      ' +
                                 str(threads))
            else:
                cron_logger.info(2 * LOG_INDENT + 'UNAVAILABLE MODE')
                return
            cron_logger.info(2 * LOG_INDENT + 'Taxpayers:    ' +
                             str(len(taxpayers)))
            cron_logger.info(2 * LOG_INDENT + 'Params:       ' +
                             str(process_params))
            if forcing_execution:
                cron_logger.info(LOG_INDENT + 'Execution forced successfully')
                print 'Performing ' + process_name + ' ... If you wish to check progress go to the log files'
            execute_with_multiprocessing(
                process_file_name=process_file_name,
                specific_process_logger=specific_process_logger,
                default_log=default_log,
                cron_logger_starting_message=cron_logger_starting_message,
                process_name=process_name,
                process_instance=process_instance,
                threads=threads,
                specific_shared_variables=specific_shared_variables,
                taxpayers=taxpayers,
                mode=forest_mode,
                forcing_execution=forcing_execution,
                process_params=process_params)
            end_time = time.time()
            process_duration = (end_time - start_time) / 3600  # in hours
            log_process_duration = False
            if len(taxpayers) > 0:
                log_process_duration = True
            if forest_mode == _Constants.SERVER_MODE and not forcing_execution:
                cron_logger.info(2 * LOG_INDENT + 'Setting server ' +
                                 str(server_index) + ' available for ' +
                                 process_name)
                _Utilities.set_process_server_available(process_name,
                                                        server_index,
                                                        logger=cron_logger)
                # BALANCER MODE is set available once all servers are available:
                all_servers_are_available = _Utilities.check_process_servers_availability(
                    process_name, logger=cron_logger)
                cron_logger.info(2 * LOG_INDENT + 'All servers availability ' +
                                 str(all_servers_are_available))
                if all_servers_are_available:
                    cron_logger.info(2 * LOG_INDENT +
                                     'Setting process available for ' +
                                     process_name)
                    if process_name == _Constants.SL1:
                        message = HAPPY_FACE + 'Ya acabe de sincronizar a todos los contribuyentes carnal y va de nuez'
                        _Utilities.send_message_to_forest_telegram_contacts(
                            message)
                    _Utilities.set_process_available(
                        process_name,
                        process_duration=process_duration,
                        logger=cron_logger,
                        log_process_duration=log_process_duration,
                        debug_execution=debug_execution)
            if forcing_execution:
                cron_logger.info(LOG_INDENT + 'Execution forced successfully')
                print 'Execution forced successfully'
                print 'You are awesome! :)'
                return
            if process_name == _Constants.EQUALIZATION:
                cron_logger.info(2 * LOG_INDENT +
                                 'Setting process available for ' +
                                 process_name)
                _Utilities.set_process_available(
                    process_name,
                    process_duration=process_duration,
                    logger=cron_logger,
                    log_process_duration=log_process_duration,
                    debug_execution=debug_execution)
        else:
            cron_logger.info(LOG_INDENT + 'End of execution')
    except Already_Handled_Exception as already_handled_exception:
        raise already_handled_exception
    except Exception as e:
        cron_logger.critical(e.message)
        already_handled_exception = Already_Handled_Exception(e.message)
        raise already_handled_exception
Beispiel #3
0
def execute(process):
	try:
		start_time = time.time()
		process_name = process['name']
		process_params = process['params']
		forcing_execution = False
		forcing_identifiers = None
		forest_mode = PROCESS_HANDLER_CONFIG['forest_mode']
		server_index = PROCESS_HANDLER_CONFIG['server_index']# The index related to this server (in case it is configured in SERVER mode)
		if process_params is not None and 'identifiers' in process_params and process_params['identifiers'] is not None:
			forcing_execution = True
			forest_mode = _Constants.SERVER_MODE# Forcing is executed as server mode
			forcing_identifiers = process_params['identifiers']
			if _Utilities.validate_forcing_identifiers(forcing_identifiers,process_name,logger=cron_logger) is False:
				print 'Identifiers are not valid, this means some identfiers do not exist, or, if they exist, their status is not equal to ' + process_name
				sys.exit()
			else:
				print 'Identifiers are OK'
		if (process_name == _Constants.INITIALIZATION or process_name == _Constants.EQUALIZATION) and forcing_execution:
			if _Utilities.validate_params(process_params,logger=cron_logger) is False:
				print 'Period params are wrong'
				sys.exit()
		debug_execution = process['debug']
		if forcing_execution is False:
			cron_logger.info(LOG_INDENT + 'Validating CRON Process')
			process_availability = _Utilities.check_process_availability('cron',debug_execution=debug_execution,logger=cron_logger)
			if process_availability is not True:
				cron_logger.info(LOG_INDENT + 'CRON is suspended')
				suspended_at = Datetime.now()
				_Utilities.update_cron_process_log('cron',logger=cron_logger,suspended_at=suspended_at,debug_execution=debug_execution)
				return False
			cron_logger.info(LOG_INDENT + 'Validating process ... ')
			process_is_valid = validate(process_name,forest_mode,server_index,debug_execution=debug_execution)
		else:
			process_is_valid = True
			print 'Forcing execution of ' + process_name
			cron_logger.info(LOG_INDENT + 'Forcing execution')
		if process_is_valid:
			cron_logger.info(LOG_INDENT + 'Updating cron db log ... ')
			if forcing_execution is False:
				_Utilities.update_cron_process_log(process_name,logger=cron_logger,debug_execution=debug_execution)
			cron_logger.info(LOG_INDENT + 'Getting instance of ' + process_name)
			# Process data:
			SPECIFIC_PROCESS_CONFIG_DATA = PROCESS_HANDLER_CONFIG[process_name]
			specific_process_logger = SPECIFIC_PROCESS_CONFIG_DATA['specific_process_logger']
			process_instance = SPECIFIC_PROCESS_CONFIG_DATA['process_instance']
			default_log = SPECIFIC_PROCESS_CONFIG_DATA['default_log']
			cron_logger_starting_message = SPECIFIC_PROCESS_CONFIG_DATA['cron_logger_starting_message']
			process_name = SPECIFIC_PROCESS_CONFIG_DATA['process_name']
			process_file_name = SPECIFIC_PROCESS_CONFIG_DATA['process_file_name']
			threads = SPECIFIC_PROCESS_CONFIG_DATA['threads']
			cron_logger.info(LOG_INDENT + 'Running in mode: ' + str(forest_mode))
			specific_shared_variables = SPECIFIC_PROCESS_CONFIG_DATA['specific_shared_variables']
			# Get process at db:
			cron_logger.info(LOG_INDENT + 'Getting process ' + process_name + ' at db')
			process = _Utilities.get_db_process(process_name)
			from_taxpayer = None
			if forcing_identifiers is not None:
				print 'Process will be forced for ' + str(len(forcing_identifiers))
				cron_logger.info(LOG_INDENT + 'This process will be force for ' + str(len(forcing_identifiers)))
			elif debug_execution is True:
				cron_logger.info(LOG_INDENT + 'This process will run in debugging mode')
			elif 'current_taxpayer' in process:
				from_taxpayer = process['current_taxpayer']# If process fails or if it is stopped it will start from this taxpayer
				cron_logger.info(LOG_INDENT + 'This process will run from taxpayer ' + from_taxpayer)
			else:
				cron_logger.info(LOG_INDENT + 'This process will run for all taxpayers')
			# Update default log
			default_log = _Utilities.add_defalut_data_to_default_log(default_log)
			# Logging:
			if debug_execution is not True:
				cron_logger.info(LOG_INDENT + 'Logging calling at cron procesess ... ')
				_Utilities.log_at_cron_processes(process)	
			cron_logger.info(LOG_INDENT + 'Getting taxpayers for this process ... ')
			taxpayers = _Utilities.get_taxpayers_for_a_specific_process(process_name,forcing_identifiers=forcing_identifiers,limit=None,from_taxpayer=from_taxpayer,logger=cron_logger,debug_execution=debug_execution,server_index=server_index,mode=forest_mode)
			# Set unavailable:
			if forest_mode == _Constants.SERVER_MODE and forcing_execution is False:
				cron_logger.info(2*LOG_INDENT + 'Setting server ' + str(server_index) + ' unavailable for ' + process_name)
				_Utilities.set_process_server_unavailable(process_name,server_index,logger=cron_logger)
			elif forest_mode == _Constants.BALANCER_MODE and forcing_execution is False:
				cron_logger.info(LOG_INDENT + 'Setting process ' + process_name + ' unavailable and could server start')
				process_availability = _Utilities.set_process_unavailable(process_name,taxpayers=taxpayers,logger=cron_logger,debug_execution=debug_execution,threads=threads)
				cron_logger.info(LOG_INDENT + process_name + ' availability: ' + str(process_availability))
			# Multi-threading execution:
			cron_logger.info(LOG_INDENT + 'Executing ... ')
			cron_logger.info(2*LOG_INDENT + 'Process name: ' + process_name)
			if forest_mode == _Constants.BALANCER_MODE:
				cron_logger.info(2*LOG_INDENT + 'Servers:      ' + str(threads))
			elif forest_mode == _Constants.SERVER_MODE:
				cron_logger.info(2*LOG_INDENT + 'Threads:      ' + str(threads))
			else:
				cron_logger.info(2*LOG_INDENT + 'UNAVAILABLE MODE')
				return
			cron_logger.info(2*LOG_INDENT + 'Taxpayers:    ' + str(len(taxpayers)))
			cron_logger.info(2*LOG_INDENT + 'Params:       ' + str(process_params))
			if forcing_execution:
				cron_logger.info(LOG_INDENT + 'Execution forced successfully')
				print 'Performing ' + process_name + ' ... If you wish to check progress go to the log files'
			execute_with_multiprocessing(process_file_name=process_file_name,specific_process_logger=specific_process_logger,default_log=default_log,cron_logger_starting_message=cron_logger_starting_message,process_name=process_name,process_instance=process_instance,threads=threads,specific_shared_variables=specific_shared_variables,taxpayers=taxpayers,mode=forest_mode,forcing_execution=forcing_execution,process_params=process_params)
			end_time = time.time()
			process_duration = (end_time - start_time)/3600# in hours
			log_process_duration = False
			if len(taxpayers) > 0:
				log_process_duration = True
			if forest_mode == _Constants.SERVER_MODE and not forcing_execution:
				cron_logger.info(2*LOG_INDENT + 'Setting server ' + str(server_index) + ' available for ' + process_name)
				_Utilities.set_process_server_available(process_name,server_index,logger=cron_logger)
				# BALANCER MODE is set available once all servers are available:
				all_servers_are_available = _Utilities.check_process_servers_availability(process_name,logger=cron_logger)
				cron_logger.info(2*LOG_INDENT + 'All servers availability ' + str(all_servers_are_available))
				if all_servers_are_available:
					cron_logger.info(2*LOG_INDENT + 'Setting process available for ' + process_name)
					if process_name == _Constants.SL1:
						message = HAPPY_FACE + 'Ya acabe de sincronizar a todos los contribuyentes carnal y va de nuez'
						_Utilities.send_message_to_forest_telegram_contacts(message)
					_Utilities.set_process_available(process_name,process_duration=process_duration,logger=cron_logger,log_process_duration=log_process_duration,debug_execution=debug_execution)
			if forcing_execution:
				cron_logger.info(LOG_INDENT + 'Execution forced successfully')
				print 'Execution forced successfully'
				print 'You are awesome! :)'
				return
			if process_name == _Constants.EQUALIZATION:
				cron_logger.info(2*LOG_INDENT + 'Setting process available for ' + process_name)
				_Utilities.set_process_available(process_name,process_duration=process_duration,logger=cron_logger,log_process_duration=log_process_duration,debug_execution=debug_execution)
		else:
			cron_logger.info(LOG_INDENT + 'End of execution')
	except Already_Handled_Exception as already_handled_exception:
		raise already_handled_exception
	except Exception as e:
		cron_logger.critical(e.message)
		already_handled_exception = Already_Handled_Exception(e.message)
		raise already_handled_exception
Beispiel #4
0
def main(argv):
    hey_execution = get_command_line_params(argv)
    cron_logger = _Utilities.setup_logger('cron')
    current_hour = Datetime.now().hour
    chat_id = None
    if hey_execution:
        hey = _Utilities.hey(logger=cron_logger, server_index=SERVER_INDEX)
        if hey is False:
            return
        chat_id = hey
    cron_logger.info(
        '------------------------------------------------------------------------'
    )
    cron_logger.info(LOG_INDENT + 'Performing Forest Supervision')
    processes = get_processes_in_this_mode()
    for process in processes:
        if process == _Constants.INITIALIZATION:
            taxpayers = _Utilities.get_taxpayers_for_a_specific_process(
                process)
            taxpayers_in_init = len(taxpayers)
            if taxpayers_in_init > 0:
                last_log_date = get_last_log_time_of_a_process(process)
            else:
                last_log_date = True
        else:
            last_log_date = get_last_log_time_of_a_process(process)
        everything_ok = is_everything_ok(last_log_date)
        cron_logger.info(LOG_INDENT + process.upper())
        current_date = Datetime.now()
        year = current_date.year
        month = current_date.month
        day = current_date.day
        begin_of_the_day = Datetime(year, month, day)
        if last_log_date > begin_of_the_day:
            last_log_date_hour = str(last_log_date)[11:11 + 8]
        else:
            last_log_date_hour = str(last_log_date)
        if everything_ok:
            message = HAPPY_FACE + 'Todo chido con ' + process + ' en Forest_' + str(
                SERVER_INDEX) + '. Tengo el ultimo registro a las ' + str(
                    last_log_date_hour)
            cron_logger.info(2 * LOG_INDENT +
                             'Everyting ok. Last log was at: ' +
                             str(last_log_date_hour))
            if chat_id is not None:
                _Utilities.send_message_to_forest_telegram_contacts(
                    message, chat_ids=[chat_id])
            else:
                _Utilities.send_message_to_forest_telegram_contacts(message)
        else:
            message = 'Tengo un ' + 3 * telegram.Emoji.PILE_OF_POO + 1 * CHANGITO + ' en Forest_' + str(
                SERVER_INDEX) + '. El ultimo registro a las ' + str(
                    last_log_date_hour)
            cron_logger.info(2 * LOG_INDENT +
                             'There is a PROBLEM. Last log was at: ' +
                             str(last_log_date_hour))
            if chat_id is not None:
                _Utilities.send_message_to_forest_telegram_contacts(
                    message, chat_ids=[chat_id])
            else:
                _Utilities.send_message_to_forest_telegram_contacts(message)