def main(argv): hey_execution = get_command_line_params(argv) cron_logger = _Utilities.setup_logger('cron') current_hour = Datetime.now().hour chat_id = None if hey_execution: hey = _Utilities.hey(logger=cron_logger,server_index=SERVER_INDEX) if hey is False: return chat_id = hey cron_logger.info('------------------------------------------------------------------------') cron_logger.info(LOG_INDENT + 'Performing Forest Supervision') processes = get_processes_in_this_mode() for process in processes: if process == _Constants.INITIALIZATION: taxpayers = _Utilities.get_taxpayers_for_a_specific_process(process) taxpayers_in_init = len(taxpayers) if taxpayers_in_init > 0: last_log_date = get_last_log_time_of_a_process(process) else: last_log_date = True else: last_log_date = get_last_log_time_of_a_process(process) everything_ok = is_everything_ok(last_log_date) cron_logger.info(LOG_INDENT + process.upper()) current_date = Datetime.now() year = current_date.year month = current_date.month day = current_date.day begin_of_the_day = Datetime(year,month,day) if last_log_date > begin_of_the_day: last_log_date_hour = str(last_log_date)[11:11+8] else: last_log_date_hour = str(last_log_date) if everything_ok: message = HAPPY_FACE + 'Todo chido con ' + process + ' en Forest_' + str(SERVER_INDEX) + '. Tengo el ultimo registro a las ' + str(last_log_date_hour) cron_logger.info(2*LOG_INDENT + 'Everyting ok. Last log was at: ' + str(last_log_date_hour)) if chat_id is not None: _Utilities.send_message_to_forest_telegram_contacts(message,chat_ids=[chat_id]) else: _Utilities.send_message_to_forest_telegram_contacts(message) else: message = 'Tengo un ' + 3*telegram.Emoji.PILE_OF_POO + 1*CHANGITO + ' en Forest_' + str(SERVER_INDEX) + '. El ultimo registro a las ' + str(last_log_date_hour) cron_logger.info(2*LOG_INDENT + 'There is a PROBLEM. Last log was at: ' + str(last_log_date_hour)) if chat_id is not None: _Utilities.send_message_to_forest_telegram_contacts(message,chat_ids=[chat_id]) else: _Utilities.send_message_to_forest_telegram_contacts(message)
def execute(process): try: start_time = time.time() process_name = process['name'] process_params = process['params'] forcing_execution = False forcing_identifiers = None forest_mode = PROCESS_HANDLER_CONFIG['forest_mode'] server_index = PROCESS_HANDLER_CONFIG[ 'server_index'] # The index related to this server (in case it is configured in SERVER mode) if process_params is not None and 'identifiers' in process_params and process_params[ 'identifiers'] is not None: forcing_execution = True forest_mode = _Constants.SERVER_MODE # Forcing is executed as server mode forcing_identifiers = process_params['identifiers'] if _Utilities.validate_forcing_identifiers( forcing_identifiers, process_name, logger=cron_logger) is False: print 'Identifiers are not valid, this means some identfiers do not exist, or, if they exist, their status is not equal to ' + process_name sys.exit() else: print 'Identifiers are OK' if (process_name == _Constants.INITIALIZATION or process_name == _Constants.EQUALIZATION) and forcing_execution: if _Utilities.validate_params(process_params, logger=cron_logger) is False: print 'Period params are wrong' sys.exit() debug_execution = process['debug'] if forcing_execution is False: cron_logger.info(LOG_INDENT + 'Validating CRON Process') process_availability = _Utilities.check_process_availability( 'cron', debug_execution=debug_execution, logger=cron_logger) if process_availability is not True: cron_logger.info(LOG_INDENT + 'CRON is suspended') suspended_at = Datetime.now() _Utilities.update_cron_process_log( 'cron', logger=cron_logger, suspended_at=suspended_at, debug_execution=debug_execution) return False cron_logger.info(LOG_INDENT + 'Validating process ... ') process_is_valid = validate(process_name, forest_mode, server_index, debug_execution=debug_execution) else: process_is_valid = True print 'Forcing execution of ' + process_name cron_logger.info(LOG_INDENT + 'Forcing execution') if process_is_valid: cron_logger.info(LOG_INDENT + 'Updating cron db log ... ') if forcing_execution is False: _Utilities.update_cron_process_log( process_name, logger=cron_logger, debug_execution=debug_execution) cron_logger.info(LOG_INDENT + 'Getting instance of ' + process_name) # Process data: SPECIFIC_PROCESS_CONFIG_DATA = PROCESS_HANDLER_CONFIG[process_name] specific_process_logger = SPECIFIC_PROCESS_CONFIG_DATA[ 'specific_process_logger'] process_instance = SPECIFIC_PROCESS_CONFIG_DATA['process_instance'] default_log = SPECIFIC_PROCESS_CONFIG_DATA['default_log'] cron_logger_starting_message = SPECIFIC_PROCESS_CONFIG_DATA[ 'cron_logger_starting_message'] process_name = SPECIFIC_PROCESS_CONFIG_DATA['process_name'] process_file_name = SPECIFIC_PROCESS_CONFIG_DATA[ 'process_file_name'] threads = SPECIFIC_PROCESS_CONFIG_DATA['threads'] cron_logger.info(LOG_INDENT + 'Running in mode: ' + str(forest_mode)) specific_shared_variables = SPECIFIC_PROCESS_CONFIG_DATA[ 'specific_shared_variables'] # Get process at db: cron_logger.info(LOG_INDENT + 'Getting process ' + process_name + ' at db') process = _Utilities.get_db_process(process_name) from_taxpayer = None if forcing_identifiers is not None: print 'Process will be forced for ' + str( len(forcing_identifiers)) cron_logger.info(LOG_INDENT + 'This process will be force for ' + str(len(forcing_identifiers))) elif debug_execution is True: cron_logger.info(LOG_INDENT + 'This process will run in debugging mode') elif 'current_taxpayer' in process: from_taxpayer = process[ 'current_taxpayer'] # If process fails or if it is stopped it will start from this taxpayer cron_logger.info(LOG_INDENT + 'This process will run from taxpayer ' + from_taxpayer) else: cron_logger.info(LOG_INDENT + 'This process will run for all taxpayers') # Update default log default_log = _Utilities.add_defalut_data_to_default_log( default_log) # Logging: if debug_execution is not True: cron_logger.info(LOG_INDENT + 'Logging calling at cron procesess ... ') _Utilities.log_at_cron_processes(process) cron_logger.info(LOG_INDENT + 'Getting taxpayers for this process ... ') taxpayers = _Utilities.get_taxpayers_for_a_specific_process( process_name, forcing_identifiers=forcing_identifiers, limit=None, from_taxpayer=from_taxpayer, logger=cron_logger, debug_execution=debug_execution, server_index=server_index, mode=forest_mode) # Set unavailable: if forest_mode == _Constants.SERVER_MODE and forcing_execution is False: cron_logger.info(2 * LOG_INDENT + 'Setting server ' + str(server_index) + ' unavailable for ' + process_name) _Utilities.set_process_server_unavailable(process_name, server_index, logger=cron_logger) elif forest_mode == _Constants.BALANCER_MODE and forcing_execution is False: cron_logger.info(LOG_INDENT + 'Setting process ' + process_name + ' unavailable and could server start') process_availability = _Utilities.set_process_unavailable( process_name, taxpayers=taxpayers, logger=cron_logger, debug_execution=debug_execution, threads=threads) cron_logger.info(LOG_INDENT + process_name + ' availability: ' + str(process_availability)) # Multi-threading execution: cron_logger.info(LOG_INDENT + 'Executing ... ') cron_logger.info(2 * LOG_INDENT + 'Process name: ' + process_name) if forest_mode == _Constants.BALANCER_MODE: cron_logger.info(2 * LOG_INDENT + 'Servers: ' + str(threads)) elif forest_mode == _Constants.SERVER_MODE: cron_logger.info(2 * LOG_INDENT + 'Threads: ' + str(threads)) else: cron_logger.info(2 * LOG_INDENT + 'UNAVAILABLE MODE') return cron_logger.info(2 * LOG_INDENT + 'Taxpayers: ' + str(len(taxpayers))) cron_logger.info(2 * LOG_INDENT + 'Params: ' + str(process_params)) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Performing ' + process_name + ' ... If you wish to check progress go to the log files' execute_with_multiprocessing( process_file_name=process_file_name, specific_process_logger=specific_process_logger, default_log=default_log, cron_logger_starting_message=cron_logger_starting_message, process_name=process_name, process_instance=process_instance, threads=threads, specific_shared_variables=specific_shared_variables, taxpayers=taxpayers, mode=forest_mode, forcing_execution=forcing_execution, process_params=process_params) end_time = time.time() process_duration = (end_time - start_time) / 3600 # in hours log_process_duration = False if len(taxpayers) > 0: log_process_duration = True if forest_mode == _Constants.SERVER_MODE and not forcing_execution: cron_logger.info(2 * LOG_INDENT + 'Setting server ' + str(server_index) + ' available for ' + process_name) _Utilities.set_process_server_available(process_name, server_index, logger=cron_logger) # BALANCER MODE is set available once all servers are available: all_servers_are_available = _Utilities.check_process_servers_availability( process_name, logger=cron_logger) cron_logger.info(2 * LOG_INDENT + 'All servers availability ' + str(all_servers_are_available)) if all_servers_are_available: cron_logger.info(2 * LOG_INDENT + 'Setting process available for ' + process_name) if process_name == _Constants.SL1: message = HAPPY_FACE + 'Ya acabe de sincronizar a todos los contribuyentes carnal y va de nuez' _Utilities.send_message_to_forest_telegram_contacts( message) _Utilities.set_process_available( process_name, process_duration=process_duration, logger=cron_logger, log_process_duration=log_process_duration, debug_execution=debug_execution) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Execution forced successfully' print 'You are awesome! :)' return if process_name == _Constants.EQUALIZATION: cron_logger.info(2 * LOG_INDENT + 'Setting process available for ' + process_name) _Utilities.set_process_available( process_name, process_duration=process_duration, logger=cron_logger, log_process_duration=log_process_duration, debug_execution=debug_execution) else: cron_logger.info(LOG_INDENT + 'End of execution') except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: cron_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def equalize_dbs_for_a_taxpayer(forcing_period=False, forcing_execution=None, taxpayer=None, process_logger=None, process_params=None): try: indent = ' ' log = {'before': {}, 'after': {'stored': 0, 'errors': 0}} # End of log # Get params: identifier = taxpayer['identifier'] created_at = taxpayer['created_at'] created_at_lower_limit = Datetime.now() - relativedelta(months=1) force_start_date = True # process_logger.info(2*LOG_INDENT + 'FORCING START DATE') if created_at > created_at_lower_limit: begin_date = taxpayer[ 'start_date'] #Since taxpayer claim to be synchronized process_logger.info(2 * LOG_INDENT + 'Chosing start date as begin date') else: current_date = Datetime.now() current_year = current_date.year begin_date = Datetime(current_year, 1, 1) # Jan 1st of the current year process_logger.info(2 * LOG_INDENT + 'Chosing ' + str(begin_date) + ' as begin date') if forcing_period: year = int(process_params['year']) month = int(process_params['month']) begin_date = Datetime(year, month, 1) begin_date = begin_date.replace(hour=0, minute=0) if 'all' in process_params: end_date = Datetime.now() # Until now else: end_date = begin_date + relativedelta(months=1) process_logger.info(2 * LOG_INDENT + 'Forcing ' + str(begin_date) + ' as begin date') process_logger.info(2 * LOG_INDENT + 'Forcing ' + str(end_date) + ' as end date') # year = str(Datetime.now().year) # months = _Utilities.get_current_fiscal_declaration_period(_Constants.TWO_MONTHS_PERIOD) # begin_date = Datetime(int(year),int(months[0]),1)# Since previous month (optimization introduced on Sep 8, 2015) begin_date = begin_date.replace(hour=0, minute=0) if not forcing_period: end_date = Datetime.now() # Until now process_logger.info(2 * LOG_INDENT + 'Equalizing dbs from ' + str(begin_date) + ' to ' + str(end_date)) # Get CFDIs from Forest DB: cfdis_in_forest_db_count = _Utilities.get_cfdis_count_in_forest_for_this_taxpayer_at_period( taxpayer, begin_date, end_date) process_logger.info(2 * LOG_INDENT + 'Retrieving ' + str(cfdis_in_forest_db_count) + ' from Forest DB ... ') cfdis_in_forest_db = _Utilities.get_cfdis_in_forest_for_this_taxpayer_at_period( taxpayer, begin_date, end_date, limit=None, logger=process_logger, dict_result=True) # Get CFDIs from Corebook DB: try: cfdis_in_corebook_db_count = _Locals.get_cfdis_count_in_corebook_for_this_taxpayer_at_period( identifier, begin_date, end_date, logger=process_logger) process_logger.info(2 * LOG_INDENT + 'Retrieving ' + str(cfdis_in_corebook_db_count) + ' from Corebook DB ... ') cfdis_in_corebook_db = _Locals.get_cfdis_in_corebook_for_this_taxpayer_at_period( identifier, begin_date, end_date, limit=None, logger=process_logger) except Already_Handled_Exception as already_handled_exception: process_logger.info(2 * LOG_INDENT + already_handled_exception.value) log['before'] = { 'forest_db': 0, 'corebook_db': 0, 'f_but_not_in_cb': 0, 'cb_but_not_in_f': 0 } # End of log log['after']['errors'] = log['after']['errors'] + 1 return log # Log data in db: log['before']['forest_db'] = len(cfdis_in_forest_db) log['before']['corebook_db'] = len(cfdis_in_corebook_db) # Get missing CFDIs in Corebook: process_logger.info(2 * LOG_INDENT + 'Getting differences in dbs ... ') # _Utilities.log_cfdis_uuids(title='Forest CFDIs: ',indent=2*LOG_INDENT,cfdis=cfdis_in_forest_db,logger=process_logger,dict=True) # _Utilities.log_cfdis_uuids(title='Corebook CFDIs: ',indent=2*LOG_INDENT,cfdis=cfdis_in_corebook_db,logger=process_logger,dict=True) missing_cfdis = _Locals.get_missing_cfdis_in_each_db( cfdis_in_forest_db, cfdis_in_corebook_db, logger=process_logger) missing_cfdis_in_corebook_db = missing_cfdis['in_corebook_db'] missing_cfdis_in_forest_db = missing_cfdis['in_forest_db'] cfdis_with_different_status = missing_cfdis[ 'cfdis_with_different_status'] # General status: log['before']['f_but_not_in_cb'] = len(missing_cfdis_in_corebook_db) log['before']['cb_but_not_in_f'] = len(missing_cfdis_in_forest_db) log['before'][ 'cfdis_with_different_status'] = cfdis_with_different_status process_logger.info(2 * LOG_INDENT + 'DBs Status: ') process_logger.info(3 * LOG_INDENT + 'Forest DB -> ' + str(log['before']['forest_db'])) process_logger.info(3 * LOG_INDENT + 'Corebook DB -> ' + str(log['before']['corebook_db'])) process_logger.info(3 * LOG_INDENT + 'F not in CB -> ' + str(log['before']['f_but_not_in_cb'] - log['before']['cfdis_with_different_status'])) process_logger.info(3 * LOG_INDENT + 'CB not in F -> ' + str(log['before']['cb_but_not_in_f'])) process_logger.info(3 * LOG_INDENT + 'Diff Status -> ' + str(log['before']['cfdis_with_different_status'])) if len(missing_cfdis_in_corebook_db) > 0: # _Utilities.log_cfdis_uuids(title='Missing CFDIs: ',indent=2*LOG_INDENT,cfdis=missing_cfdis_in_corebook_db,logger=process_logger) # log['after']['stored'] = len(missing_cfdis_in_corebook_db) # log['after']['errors'] = 0 cb_summary = _Locals.store_missing_cfdis_in_corebook( missing_cfdis_in_corebook_db, identifier, logger=process_logger, limit=None) log['after']['stored'] = cb_summary['stored'] log['after']['errors'] = cb_summary['errors'] process_logger.info(2 * LOG_INDENT + 'Equalization Summary: ') process_logger.info(3 * LOG_INDENT + 'CFDIs stored in CB -> ' + str(log['after']['stored'])) process_logger.info(3 * LOG_INDENT + 'Errors occurred -> ' + str(log['after']['errors'])) if forcing_execution: process_logger.info(3 * LOG_INDENT + 'Sending telegram notification ... ') message = 'Ya iguale a este vato: ' + taxpayer['identifier'] _Utilities.send_message_to_forest_telegram_contacts( message, logger=process_logger) return log except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: equalization_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def excute_initialization_for_taxpayer(forcing_execution=False, forcing_period=False, taxpayer=None, process_logger=None, process_params=None): try: process_logger.info(2 * LOG_INDENT + 'Forcing period: ' + str(forcing_period)) initialization_data = _Locals.get_initialization_data( taxpayer, logger=process_logger, process_params=process_params) initialized = initialization_data['initialized'] if initialized == False: initialization_log = _Locals.new_initialization_log( logger=process_logger) # Get CFDIs from DB: process_logger.info(2 * LOG_INDENT + 'RETRIEVING DATA FROM FOREST DB ... ') _year = str(initialization_data['year']) _month = str(initialization_data['month']) process_logger.info(3 * LOG_INDENT + 'Year: ' + str(initialization_data['year']) + ' Month: ' + str(initialization_data['month'])) process_logger.info(3 * LOG_INDENT + 'From ' + str(initialization_data['begin_date']) + ' to ' + str(initialization_data['end_date'])) cfdis_in_db = _Utilities.get_cfdis_in_forest_for_this_taxpayer_at_period( taxpayer, initialization_data['begin_date'], initialization_data['end_date'], limit=None) # Manage or format CFDis data: existing_cfdi_uuids = _Utilities.get_existing_uuids_in_forest_db( cfdis_in_db=cfdis_in_db, logger=process_logger) # Log Forest data: process_logger.info(3 * LOG_INDENT + 'Existing: ' + str(len(existing_cfdi_uuids))) # Get CFDIs from firmware: get_sat_updates_params = { 'identifier': taxpayer['identifier'], 'password': taxpayer['password'], 'year': initialization_data['year'], 'months': [initialization_data['month']], 'uuids': existing_cfdi_uuids } # End of get_sat_update_params # ------------------------------------------------------------------- # bugSolved 12/Ago/15 # Event: timeout value was becoming longer and longer because of connection problems (firmware servers could not be reached due to connection problems instead of logic problems) # firmware_timeout = taxpayer['firmware_timeout'] if 'firmware_timeout' in taxpayer and taxpayer['firmware_timeout'] is not None else _Constants.DEFAULT_FIRMWARE_TIMEOUT firmware_timeout = _Constants.DEFAULT_FIRMWARE_TIMEOUT # ------------------------------------------------------------------- # process_logger.info(2*LOG_INDENT + 'RETRIEVING DATA FROM FIRMWARE (SAT) constant timeout = ' + str(firmware_timeout) + ' secs') process_logger.info(2 * LOG_INDENT + 'RETRIEVING DATA FROM FIRMWARE (SAT)') sat_updates = _Firmware.isa(instruction='get_sat_updates', params=get_sat_updates_params, log=initialization_log, logger=process_logger, timeout=firmware_timeout, taxpayer=taxpayer) new_cfdis = sat_updates['new'] process_logger.info(3 * LOG_INDENT + 'CFDI new: ' + str(initialization_log['firmware']['new'])) process_logger.info(3 * LOG_INDENT + 'CFDI to-update: ' + str(initialization_log['firmware']['update'])) # Update Forest DB -> NEW OR COMPLETED: process_logger.info(2 * LOG_INDENT + 'UPDATING FOREST DB ... ') n = 0 forest_db = _Utilities.set_connection_to_forest_db() for new_cfdi in new_cfdis: try: uuid = new_cfdi['uuid'] _Utilities.create_cfdi(new_cfdi, logger=process_logger, log=initialization_log, forest_db=forest_db) n = n + 1 process_logger.info(3 * LOG_INDENT + str(n) + '. ' + uuid + ' stored in Forest DB') except: process_logger.info( 3 * LOG_INDENT + str(n) + '. ' + uuid + ' could not be stored in Forest DB (ERROR)') process_logger.info(2 * LOG_INDENT + 'SUMMARY ... ') process_logger.info( 3 * LOG_INDENT + 'New stored: ' + str(initialization_log['forest_db']['after']['new'])) process_logger.info( 3 * LOG_INDENT + 'Pending: ' + str(initialization_log['forest_db']['after']['pending'])) initialization_result = { 'new': initialization_log['firmware']['new'], 'stored': initialization_log['forest_db']['after']['new'], 'year_initialized': initialization_data['year'], 'month_initialized': initialization_data['month'], 'avoid_iteration': initialization_log['avoid_iteration'] if 'avoid_iteration' in initialization_log else False } # End of initialization_result else: initialization_result = { 'new': _Constants.ZLATAN, 'stored': _Constants.ZLATAN, 'year_initialized': _Constants.ZLATAN, 'month_initialized': _Constants.ZLATAN } # End of initialization_result # Update taxpayer: new_initialization_data = initialization_data[ 'new_initialization_data'] initialization_result['initialized'] = initialized if forcing_period or ('avoid_iteration' in initialization_result and initialization_result['avoid_iteration'] == True): process_logger.info( 2 * LOG_INDENT + 'NOT updating taxpayer initialization status ... ') initialization_result['percentage_initialized'] = "--.-%" else: taxpayer = _Locals.update_taxpayer_initialization_status( taxpayer, new_initialization_data, logger=process_logger, initialized=initialized) initialization_result['percentage_initialized'] = taxpayer['data'][ 'percentage_initialized'] process_logger.info( 3 * LOG_INDENT + 'Percentage initialized: ' + str(initialization_result['percentage_initialized'])) if forcing_execution: process_logger.info(3 * LOG_INDENT + 'Sending telegram notification ... ') message = 'Ya inicialice a este vato: ' + taxpayer[ 'identifier'] + ' para el periodo ' + _month + '/' + _year _Utilities.send_message_to_forest_telegram_contacts( message, logger=process_logger) return initialization_result except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: process_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def execute(process): try: start_time = time.time() process_name = process['name'] process_params = process['params'] forcing_execution = False forcing_identifiers = None forest_mode = PROCESS_HANDLER_CONFIG['forest_mode'] server_index = PROCESS_HANDLER_CONFIG['server_index']# The index related to this server (in case it is configured in SERVER mode) if process_params is not None and 'identifiers' in process_params and process_params['identifiers'] is not None: forcing_execution = True forest_mode = _Constants.SERVER_MODE# Forcing is executed as server mode forcing_identifiers = process_params['identifiers'] if _Utilities.validate_forcing_identifiers(forcing_identifiers,process_name,logger=cron_logger) is False: print 'Identifiers are not valid, this means some identfiers do not exist, or, if they exist, their status is not equal to ' + process_name sys.exit() else: print 'Identifiers are OK' if (process_name == _Constants.INITIALIZATION or process_name == _Constants.EQUALIZATION) and forcing_execution: if _Utilities.validate_params(process_params,logger=cron_logger) is False: print 'Period params are wrong' sys.exit() debug_execution = process['debug'] if forcing_execution is False: cron_logger.info(LOG_INDENT + 'Validating CRON Process') process_availability = _Utilities.check_process_availability('cron',debug_execution=debug_execution,logger=cron_logger) if process_availability is not True: cron_logger.info(LOG_INDENT + 'CRON is suspended') suspended_at = Datetime.now() _Utilities.update_cron_process_log('cron',logger=cron_logger,suspended_at=suspended_at,debug_execution=debug_execution) return False cron_logger.info(LOG_INDENT + 'Validating process ... ') process_is_valid = validate(process_name,forest_mode,server_index,debug_execution=debug_execution) else: process_is_valid = True print 'Forcing execution of ' + process_name cron_logger.info(LOG_INDENT + 'Forcing execution') if process_is_valid: cron_logger.info(LOG_INDENT + 'Updating cron db log ... ') if forcing_execution is False: _Utilities.update_cron_process_log(process_name,logger=cron_logger,debug_execution=debug_execution) cron_logger.info(LOG_INDENT + 'Getting instance of ' + process_name) # Process data: SPECIFIC_PROCESS_CONFIG_DATA = PROCESS_HANDLER_CONFIG[process_name] specific_process_logger = SPECIFIC_PROCESS_CONFIG_DATA['specific_process_logger'] process_instance = SPECIFIC_PROCESS_CONFIG_DATA['process_instance'] default_log = SPECIFIC_PROCESS_CONFIG_DATA['default_log'] cron_logger_starting_message = SPECIFIC_PROCESS_CONFIG_DATA['cron_logger_starting_message'] process_name = SPECIFIC_PROCESS_CONFIG_DATA['process_name'] process_file_name = SPECIFIC_PROCESS_CONFIG_DATA['process_file_name'] threads = SPECIFIC_PROCESS_CONFIG_DATA['threads'] cron_logger.info(LOG_INDENT + 'Running in mode: ' + str(forest_mode)) specific_shared_variables = SPECIFIC_PROCESS_CONFIG_DATA['specific_shared_variables'] # Get process at db: cron_logger.info(LOG_INDENT + 'Getting process ' + process_name + ' at db') process = _Utilities.get_db_process(process_name) from_taxpayer = None if forcing_identifiers is not None: print 'Process will be forced for ' + str(len(forcing_identifiers)) cron_logger.info(LOG_INDENT + 'This process will be force for ' + str(len(forcing_identifiers))) elif debug_execution is True: cron_logger.info(LOG_INDENT + 'This process will run in debugging mode') elif 'current_taxpayer' in process: from_taxpayer = process['current_taxpayer']# If process fails or if it is stopped it will start from this taxpayer cron_logger.info(LOG_INDENT + 'This process will run from taxpayer ' + from_taxpayer) else: cron_logger.info(LOG_INDENT + 'This process will run for all taxpayers') # Update default log default_log = _Utilities.add_defalut_data_to_default_log(default_log) # Logging: if debug_execution is not True: cron_logger.info(LOG_INDENT + 'Logging calling at cron procesess ... ') _Utilities.log_at_cron_processes(process) cron_logger.info(LOG_INDENT + 'Getting taxpayers for this process ... ') taxpayers = _Utilities.get_taxpayers_for_a_specific_process(process_name,forcing_identifiers=forcing_identifiers,limit=None,from_taxpayer=from_taxpayer,logger=cron_logger,debug_execution=debug_execution,server_index=server_index,mode=forest_mode) # Set unavailable: if forest_mode == _Constants.SERVER_MODE and forcing_execution is False: cron_logger.info(2*LOG_INDENT + 'Setting server ' + str(server_index) + ' unavailable for ' + process_name) _Utilities.set_process_server_unavailable(process_name,server_index,logger=cron_logger) elif forest_mode == _Constants.BALANCER_MODE and forcing_execution is False: cron_logger.info(LOG_INDENT + 'Setting process ' + process_name + ' unavailable and could server start') process_availability = _Utilities.set_process_unavailable(process_name,taxpayers=taxpayers,logger=cron_logger,debug_execution=debug_execution,threads=threads) cron_logger.info(LOG_INDENT + process_name + ' availability: ' + str(process_availability)) # Multi-threading execution: cron_logger.info(LOG_INDENT + 'Executing ... ') cron_logger.info(2*LOG_INDENT + 'Process name: ' + process_name) if forest_mode == _Constants.BALANCER_MODE: cron_logger.info(2*LOG_INDENT + 'Servers: ' + str(threads)) elif forest_mode == _Constants.SERVER_MODE: cron_logger.info(2*LOG_INDENT + 'Threads: ' + str(threads)) else: cron_logger.info(2*LOG_INDENT + 'UNAVAILABLE MODE') return cron_logger.info(2*LOG_INDENT + 'Taxpayers: ' + str(len(taxpayers))) cron_logger.info(2*LOG_INDENT + 'Params: ' + str(process_params)) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Performing ' + process_name + ' ... If you wish to check progress go to the log files' execute_with_multiprocessing(process_file_name=process_file_name,specific_process_logger=specific_process_logger,default_log=default_log,cron_logger_starting_message=cron_logger_starting_message,process_name=process_name,process_instance=process_instance,threads=threads,specific_shared_variables=specific_shared_variables,taxpayers=taxpayers,mode=forest_mode,forcing_execution=forcing_execution,process_params=process_params) end_time = time.time() process_duration = (end_time - start_time)/3600# in hours log_process_duration = False if len(taxpayers) > 0: log_process_duration = True if forest_mode == _Constants.SERVER_MODE and not forcing_execution: cron_logger.info(2*LOG_INDENT + 'Setting server ' + str(server_index) + ' available for ' + process_name) _Utilities.set_process_server_available(process_name,server_index,logger=cron_logger) # BALANCER MODE is set available once all servers are available: all_servers_are_available = _Utilities.check_process_servers_availability(process_name,logger=cron_logger) cron_logger.info(2*LOG_INDENT + 'All servers availability ' + str(all_servers_are_available)) if all_servers_are_available: cron_logger.info(2*LOG_INDENT + 'Setting process available for ' + process_name) if process_name == _Constants.SL1: message = HAPPY_FACE + 'Ya acabe de sincronizar a todos los contribuyentes carnal y va de nuez' _Utilities.send_message_to_forest_telegram_contacts(message) _Utilities.set_process_available(process_name,process_duration=process_duration,logger=cron_logger,log_process_duration=log_process_duration,debug_execution=debug_execution) if forcing_execution: cron_logger.info(LOG_INDENT + 'Execution forced successfully') print 'Execution forced successfully' print 'You are awesome! :)' return if process_name == _Constants.EQUALIZATION: cron_logger.info(2*LOG_INDENT + 'Setting process available for ' + process_name) _Utilities.set_process_available(process_name,process_duration=process_duration,logger=cron_logger,log_process_duration=log_process_duration,debug_execution=debug_execution) else: cron_logger.info(LOG_INDENT + 'End of execution') except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: cron_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def excute_initialization_for_taxpayer(forcing_execution=False,forcing_period=False,taxpayer=None,process_logger=None,process_params=None): try: process_logger.info(2*LOG_INDENT + 'Forcing period: ' + str(forcing_period)) initialization_data = _Locals.get_initialization_data(taxpayer,logger=process_logger,process_params=process_params) initialized = initialization_data['initialized'] if initialized == False: initialization_log = _Locals.new_initialization_log(logger=process_logger) # Get CFDIs from DB: process_logger.info(2*LOG_INDENT + 'RETRIEVING DATA FROM FOREST DB ... ') _year = str(initialization_data['year']) _month = str(initialization_data['month']) process_logger.info(3*LOG_INDENT + 'Year: ' + str(initialization_data['year']) + ' Month: ' + str(initialization_data['month'])) process_logger.info(3*LOG_INDENT + 'From ' + str(initialization_data['begin_date']) + ' to ' + str(initialization_data['end_date'])) cfdis_in_db = _Utilities.get_cfdis_in_forest_for_this_taxpayer_at_period(taxpayer,initialization_data['begin_date'],initialization_data['end_date'],limit=None) # Manage or format CFDis data: existing_cfdi_uuids = _Utilities.get_existing_uuids_in_forest_db(cfdis_in_db=cfdis_in_db,logger=process_logger) # Log Forest data: process_logger.info(3*LOG_INDENT + 'Existing: ' + str(len(existing_cfdi_uuids))) # Get CFDIs from firmware: get_sat_updates_params = { 'identifier' : taxpayer['identifier'], 'password' : taxpayer['password'], 'year' : initialization_data['year'], 'months' : [initialization_data['month']], 'uuids' : existing_cfdi_uuids }# End of get_sat_update_params # ------------------------------------------------------------------- # bugSolved 12/Ago/15 # Event: timeout value was becoming longer and longer because of connection problems (firmware servers could not be reached due to connection problems instead of logic problems) # firmware_timeout = taxpayer['firmware_timeout'] if 'firmware_timeout' in taxpayer and taxpayer['firmware_timeout'] is not None else _Constants.DEFAULT_FIRMWARE_TIMEOUT firmware_timeout = _Constants.DEFAULT_FIRMWARE_TIMEOUT # ------------------------------------------------------------------- # process_logger.info(2*LOG_INDENT + 'RETRIEVING DATA FROM FIRMWARE (SAT) constant timeout = ' + str(firmware_timeout) + ' secs') process_logger.info(2*LOG_INDENT + 'RETRIEVING DATA FROM FIRMWARE (SAT)') sat_updates = _Firmware.isa(instruction='get_sat_updates',params=get_sat_updates_params,log=initialization_log,logger=process_logger,timeout=firmware_timeout,taxpayer=taxpayer) new_cfdis = sat_updates['new'] process_logger.info(3*LOG_INDENT + 'CFDI new: ' + str(initialization_log['firmware']['new'])) process_logger.info(3*LOG_INDENT + 'CFDI to-update: ' + str(initialization_log['firmware']['update'])) # Update Forest DB -> NEW OR COMPLETED: process_logger.info(2*LOG_INDENT + 'UPDATING FOREST DB ... ') n = 0 forest_db = _Utilities.set_connection_to_forest_db() for new_cfdi in new_cfdis: try: uuid = new_cfdi['uuid'] _Utilities.create_cfdi(new_cfdi,logger=process_logger,log=initialization_log,forest_db=forest_db) n = n + 1 process_logger.info(3*LOG_INDENT + str(n) + '. ' + uuid + ' stored in Forest DB') except: process_logger.info(3*LOG_INDENT + str(n) + '. ' + uuid + ' could not be stored in Forest DB (ERROR)') process_logger.info(2*LOG_INDENT + 'SUMMARY ... ') process_logger.info(3*LOG_INDENT + 'New stored: ' + str(initialization_log['forest_db']['after']['new'])) process_logger.info(3*LOG_INDENT + 'Pending: ' + str(initialization_log['forest_db']['after']['pending'])) initialization_result = { 'new' : initialization_log['firmware']['new'], 'stored' : initialization_log['forest_db']['after']['new'], 'year_initialized' : initialization_data['year'], 'month_initialized' : initialization_data['month'], 'avoid_iteration' : initialization_log['avoid_iteration'] if 'avoid_iteration' in initialization_log else False }# End of initialization_result else: initialization_result = { 'new' : _Constants.ZLATAN, 'stored' : _Constants.ZLATAN, 'year_initialized' : _Constants.ZLATAN, 'month_initialized' : _Constants.ZLATAN }# End of initialization_result # Update taxpayer: new_initialization_data = initialization_data['new_initialization_data'] initialization_result['initialized'] = initialized if forcing_period or ('avoid_iteration' in initialization_result and initialization_result['avoid_iteration'] == True): process_logger.info(2*LOG_INDENT + 'NOT updating taxpayer initialization status ... ') initialization_result['percentage_initialized'] = "--.-%" else: taxpayer = _Locals.update_taxpayer_initialization_status(taxpayer,new_initialization_data,logger=process_logger,initialized=initialized) initialization_result['percentage_initialized'] = taxpayer['data']['percentage_initialized'] process_logger.info(3*LOG_INDENT + 'Percentage initialized: ' + str(initialization_result['percentage_initialized'])) if forcing_execution: process_logger.info(3*LOG_INDENT + 'Sending telegram notification ... ') message = 'Ya inicialice a este vato: ' + taxpayer['identifier'] + ' para el periodo ' + _month + '/' + _year _Utilities.send_message_to_forest_telegram_contacts(message,logger=process_logger) return initialization_result except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: process_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def equalize_dbs_for_a_taxpayer(forcing_period=False,forcing_execution=None,taxpayer=None,process_logger=None,process_params=None): try: indent = ' ' log = { 'before' : {}, 'after' : { 'stored' : 0, 'errors' : 0 } }# End of log # Get params: identifier = taxpayer['identifier'] created_at = taxpayer['created_at'] created_at_lower_limit = Datetime.now() - relativedelta(months=1) force_start_date = True # process_logger.info(2*LOG_INDENT + 'FORCING START DATE') if created_at > created_at_lower_limit: begin_date = taxpayer['start_date']#Since taxpayer claim to be synchronized process_logger.info(2*LOG_INDENT + 'Chosing start date as begin date') else: current_date = Datetime.now() current_year = current_date.year begin_date = Datetime(current_year,1,1)# Jan 1st of the current year process_logger.info(2*LOG_INDENT + 'Chosing ' + str(begin_date) + ' as begin date') if forcing_period: year = int(process_params['year']) month = int(process_params['month']) begin_date = Datetime(year,month,1) begin_date = begin_date.replace(hour=0, minute=0) if 'all' in process_params: end_date = Datetime.now()# Until now else: end_date = begin_date + relativedelta(months=1) process_logger.info(2*LOG_INDENT + 'Forcing ' + str(begin_date) + ' as begin date') process_logger.info(2*LOG_INDENT + 'Forcing ' + str(end_date) + ' as end date') # year = str(Datetime.now().year) # months = _Utilities.get_current_fiscal_declaration_period(_Constants.TWO_MONTHS_PERIOD) # begin_date = Datetime(int(year),int(months[0]),1)# Since previous month (optimization introduced on Sep 8, 2015) begin_date = begin_date.replace(hour=0, minute=0) if not forcing_period: end_date = Datetime.now()# Until now process_logger.info(2*LOG_INDENT + 'Equalizing dbs from ' + str(begin_date) + ' to ' + str(end_date)) # Get CFDIs from Forest DB: cfdis_in_forest_db_count = _Utilities.get_cfdis_count_in_forest_for_this_taxpayer_at_period(taxpayer,begin_date,end_date) process_logger.info(2*LOG_INDENT + 'Retrieving ' + str(cfdis_in_forest_db_count) + ' from Forest DB ... ') cfdis_in_forest_db = _Utilities.get_cfdis_in_forest_for_this_taxpayer_at_period(taxpayer,begin_date,end_date,limit=None,logger=process_logger,dict_result=True) # Get CFDIs from Corebook DB: try: cfdis_in_corebook_db_count = _Locals.get_cfdis_count_in_corebook_for_this_taxpayer_at_period(identifier,begin_date,end_date,logger=process_logger) process_logger.info(2*LOG_INDENT + 'Retrieving ' + str(cfdis_in_corebook_db_count) + ' from Corebook DB ... ') cfdis_in_corebook_db = _Locals.get_cfdis_in_corebook_for_this_taxpayer_at_period(identifier,begin_date,end_date,limit=None,logger=process_logger) except Already_Handled_Exception as already_handled_exception: process_logger.info(2*LOG_INDENT + already_handled_exception.value) log['before'] = { 'forest_db' : 0, 'corebook_db' : 0, 'f_but_not_in_cb' : 0, 'cb_but_not_in_f' : 0 }# End of log log['after']['errors'] = log['after']['errors'] + 1 return log # Log data in db: log['before']['forest_db'] = len(cfdis_in_forest_db) log['before']['corebook_db'] = len(cfdis_in_corebook_db) # Get missing CFDIs in Corebook: process_logger.info(2*LOG_INDENT + 'Getting differences in dbs ... ') # _Utilities.log_cfdis_uuids(title='Forest CFDIs: ',indent=2*LOG_INDENT,cfdis=cfdis_in_forest_db,logger=process_logger,dict=True) # _Utilities.log_cfdis_uuids(title='Corebook CFDIs: ',indent=2*LOG_INDENT,cfdis=cfdis_in_corebook_db,logger=process_logger,dict=True) missing_cfdis = _Locals.get_missing_cfdis_in_each_db(cfdis_in_forest_db,cfdis_in_corebook_db,logger=process_logger) missing_cfdis_in_corebook_db = missing_cfdis['in_corebook_db'] missing_cfdis_in_forest_db = missing_cfdis['in_forest_db'] cfdis_with_different_status = missing_cfdis['cfdis_with_different_status'] # General status: log['before']['f_but_not_in_cb'] = len(missing_cfdis_in_corebook_db) log['before']['cb_but_not_in_f'] = len(missing_cfdis_in_forest_db) log['before']['cfdis_with_different_status'] = cfdis_with_different_status process_logger.info(2*LOG_INDENT + 'DBs Status: ') process_logger.info(3*LOG_INDENT + 'Forest DB -> ' + str(log['before']['forest_db'])) process_logger.info(3*LOG_INDENT + 'Corebook DB -> ' + str(log['before']['corebook_db'])) process_logger.info(3*LOG_INDENT + 'F not in CB -> ' + str(log['before']['f_but_not_in_cb'] - log['before']['cfdis_with_different_status'])) process_logger.info(3*LOG_INDENT + 'CB not in F -> ' + str(log['before']['cb_but_not_in_f'])) process_logger.info(3*LOG_INDENT + 'Diff Status -> ' + str(log['before']['cfdis_with_different_status'])) if len(missing_cfdis_in_corebook_db) > 0: # _Utilities.log_cfdis_uuids(title='Missing CFDIs: ',indent=2*LOG_INDENT,cfdis=missing_cfdis_in_corebook_db,logger=process_logger) # log['after']['stored'] = len(missing_cfdis_in_corebook_db) # log['after']['errors'] = 0 cb_summary = _Locals.store_missing_cfdis_in_corebook(missing_cfdis_in_corebook_db,identifier,logger=process_logger,limit=None) log['after']['stored'] = cb_summary['stored'] log['after']['errors'] = cb_summary['errors'] process_logger.info(2*LOG_INDENT + 'Equalization Summary: ') process_logger.info(3*LOG_INDENT + 'CFDIs stored in CB -> ' + str(log['after']['stored'])) process_logger.info(3*LOG_INDENT + 'Errors occurred -> ' + str(log['after']['errors'])) if forcing_execution: process_logger.info(3*LOG_INDENT + 'Sending telegram notification ... ') message = 'Ya iguale a este vato: ' + taxpayer['identifier'] _Utilities.send_message_to_forest_telegram_contacts(message,logger=process_logger) return log except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: equalization_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception
def main(argv): hey_execution = get_command_line_params(argv) cron_logger = _Utilities.setup_logger('cron') current_hour = Datetime.now().hour chat_id = None if hey_execution: hey = _Utilities.hey(logger=cron_logger, server_index=SERVER_INDEX) if hey is False: return chat_id = hey cron_logger.info( '------------------------------------------------------------------------' ) cron_logger.info(LOG_INDENT + 'Performing Forest Supervision') processes = get_processes_in_this_mode() for process in processes: if process == _Constants.INITIALIZATION: taxpayers = _Utilities.get_taxpayers_for_a_specific_process( process) taxpayers_in_init = len(taxpayers) if taxpayers_in_init > 0: last_log_date = get_last_log_time_of_a_process(process) else: last_log_date = True else: last_log_date = get_last_log_time_of_a_process(process) everything_ok = is_everything_ok(last_log_date) cron_logger.info(LOG_INDENT + process.upper()) current_date = Datetime.now() year = current_date.year month = current_date.month day = current_date.day begin_of_the_day = Datetime(year, month, day) if last_log_date > begin_of_the_day: last_log_date_hour = str(last_log_date)[11:11 + 8] else: last_log_date_hour = str(last_log_date) if everything_ok: message = HAPPY_FACE + 'Todo chido con ' + process + ' en Forest_' + str( SERVER_INDEX) + '. Tengo el ultimo registro a las ' + str( last_log_date_hour) cron_logger.info(2 * LOG_INDENT + 'Everyting ok. Last log was at: ' + str(last_log_date_hour)) if chat_id is not None: _Utilities.send_message_to_forest_telegram_contacts( message, chat_ids=[chat_id]) else: _Utilities.send_message_to_forest_telegram_contacts(message) else: message = 'Tengo un ' + 3 * telegram.Emoji.PILE_OF_POO + 1 * CHANGITO + ' en Forest_' + str( SERVER_INDEX) + '. El ultimo registro a las ' + str( last_log_date_hour) cron_logger.info(2 * LOG_INDENT + 'There is a PROBLEM. Last log was at: ' + str(last_log_date_hour)) if chat_id is not None: _Utilities.send_message_to_forest_telegram_contacts( message, chat_ids=[chat_id]) else: _Utilities.send_message_to_forest_telegram_contacts(message)
def excute_synchronization_layer_1_for_taxpayer(forcing_execution=False, taxpayer=None, sl1_data=None, process_logger=None): try: sl1_execution_log = _Locals.new_synchronization_layer_1_log() # Get CFDIs from DB: process_logger.info(2 * LOG_INDENT + 'RETRIEVING DATA FROM FOREST DB ... ') cfdis_in_db = _Utilities.get_cfdis_in_forest_for_this_taxpayer_at_period( taxpayer, sl1_data['begin_date'], sl1_data['end_date']) pending_cfdis_in_db = _Utilities.get_pending_cfdis_in_forest_for_this_taxpayer_at_period( taxpayer, sl1_data['begin_date'], sl1_data['end_date']) # Manage or format CFDis data: existing_cfdi_uuids = _Locals.get_existing_uuids_in_forest_db( cfdis_in_db=cfdis_in_db, logger=process_logger, sl1_execution_log=sl1_execution_log) pending_cfdi_uuids = _Locals.get_pending_uuids_in_forest_db( pending_cfdis_in_db=pending_cfdis_in_db, logger=process_logger, sl1_execution_log=sl1_execution_log) # Log Forest data: process_logger.info( 3 * LOG_INDENT + 'Existing: ' + str(sl1_execution_log['forest_db']['before']['good'])) process_logger.info( 3 * LOG_INDENT + 'Pending: ' + str(sl1_execution_log['forest_db']['before']['pending'])) # Get CFDIs from firmware: get_sat_updates_params = { 'identifier': taxpayer['identifier'], 'password': taxpayer['password'], 'year': sl1_data['year'], 'months': sl1_data['months'], 'uuids': existing_cfdi_uuids } # End of get_sat_update_params # ------------------------------------------------------------------- # bugSolved 12/Ago/15 # Event: timeout value was becoming longer and longer because of connection problems (firmware servers could not be reached due to connection problems instead of logic problems) # firmware_timeout = taxpayer['firmware_timeout'] if 'firmware_timeout' in taxpayer and taxpayer['firmware_timeout'] is not None else _Constants.DEFAULT_FIRMWARE_TIMEOUT firmware_timeout = _Constants.DEFAULT_FIRMWARE_TIMEOUT # ------------------------------------------------------------------- # process_logger.info(2*LOG_INDENT + 'RETRIEVING DATA FROM FIRMWARE (SAT) constant timeout = ' + str(firmware_timeout) + ' secs') process_logger.info(2 * LOG_INDENT + 'RETRIEVING DATA FROM FIRMWARE (SAT)') sat_updates = _Firmware.isa(instruction='get_sat_updates', params=get_sat_updates_params, log=sl1_execution_log, logger=process_logger, timeout=firmware_timeout, taxpayer=taxpayer) new_cfdis = sat_updates['new'] updated_cfdis = sat_updates['updated'] process_logger.info(3 * LOG_INDENT + 'CFDI new: ' + str(sl1_execution_log['firmware']['new'])) process_logger.info(3 * LOG_INDENT + 'CFDI to-update: ' + str(sl1_execution_log['firmware']['update'])) # Update Forest DB -> NEW OR COMPLETED: process_logger.info(2 * LOG_INDENT + 'UPDATING FOREST DB (creting new CFDIs) ... ') n = 0 for new_cfdi in new_cfdis: n = n + 1 uuid = new_cfdi['uuid'] if uuid in pending_cfdi_uuids: # Complete pending cfdis: if 'xml' in new_cfdi and new_cfdi['xml'] != '': cfdi_with_missing_xml = new_cfdi _Locals.set_xml_to_pending_cfdi( cfdi_with_missing_xml, logger=process_logger, sl1_execution_log=sl1_execution_log) process_logger.info(3 * LOG_INDENT + str(n) + '. ' + uuid + ' completed in Forest DB') else: # Completely new ones: try: _Utilities.create_cfdi(new_cfdi, logger=process_logger, log=sl1_execution_log) process_logger.info(3 * LOG_INDENT + str(n) + '. ' + uuid + ' stored in Forest DB') except Exception as e: process_logger.info(3 * LOG_INDENT + str(n) + '. Exception catched with this one') process_logger.info(str(e)) # Update Forest DB -> JUST UPDATING: process_logger.info( 2 * LOG_INDENT + 'UPDATING FOREST DB (updating existing CFDIs) ... ') cfdis_in_db = _Utilities.get_cfdis_in_forest_for_this_taxpayer_at_period( taxpayer, sl1_data['begin_date'], sl1_data['end_date'] ) # Get cfdis for updating data (They must be retrieved again due to cursor invalidation problems): updated_cfdi_uuids = [] updated_cfdis_status_by_uuid = {} cfdis_with_status_updated = 0 default_canceled_xml_inserted_in_updating = 0 n = 0 for updated_cfdi in updated_cfdis: for cfdi_in_db in cfdis_in_db: cfdi_in_db_uuid = cfdi_in_db['uuid'] updating_cfdi_uuid = updated_cfdi['uuid'] if cfdi_in_db_uuid == updating_cfdi_uuid: updated_cfdi_status = updated_cfdi['status'] # Compare the one in forest db and the one in SAT to equalize CFDI state: if updated_cfdi_status == _Constants.CANCELED_STATUS: _Locals.set_cancelled_status_to_cfdi( cfdi_in_db, updated_cfdi, logger=process_logger, sl1_execution_log=sl1_execution_log) n = n + 1 process_logger.info(3 * LOG_INDENT + str(n) + '. ' + cfdi_in_db_uuid + ' updated in Forest DB') process_logger.info(2 * LOG_INDENT + 'SUMMARY ... ') process_logger.info( 3 * LOG_INDENT + 'New stored: ' + str(sl1_execution_log['forest_db']['after']['new'])) process_logger.info( 3 * LOG_INDENT + 'Pending completed: ' + str(sl1_execution_log['forest_db']['after']['pending_completed'])) process_logger.info( 3 * LOG_INDENT + 'Pending: ' + str(sl1_execution_log['forest_db']['after']['pending'])) process_logger.info( 3 * LOG_INDENT + 'Updated: ' + str(sl1_execution_log['forest_db']['after']['updated'])) if forcing_execution: process_logger.info(3 * LOG_INDENT + 'Sending telegram notification ... ') message = 'Ya sincronice a este vato: ' + taxpayer['identifier'] _Utilities.send_message_to_forest_telegram_contacts( message, logger=process_logger) return sl1_execution_log except Already_Handled_Exception as already_handled_exception: raise already_handled_exception except Exception as e: process_logger.critical(e.message) already_handled_exception = Already_Handled_Exception(e.message) raise already_handled_exception