def main_steps(config, output_columns_map): while True: try: message = Mail(config) message.save_earlier_mail_attachment() input = Input(config, file_path=message.saved_attachment) search_names = input.parse_input(message) break except Exception as exception: delete_temp(config) if '*Failed_message' not in exception.args: raise exception # unexpected error if len(search_names) > 0: logger.info(config.get('info', 'scrapping')) try: scrap_result = get_forms_df(config, search_names, output_columns_map) except Exception: logger.error(config.get('error', 'fpds_site')) message.send_fail_to_admin(letter=3) raise Exception('*Handled_error') input.add_status_column(scrap_result['status_res']) if not scrap_result['output_df'].empty: output = Output(config) output.write(scrap_result['output_df']) message.success_reply(output_file=output.path, no_processed=input.has_non_processed) return message.success_reply(output_file=None, no_processed=input.has_non_processed)
def __bot_process(self): while True: try: message = Mail(self.config) message.save_earlier_mail_attachment() input = Input(self.config, file_path=message.saved_attachment) search_names = input.parse_input(message) break except Exception as exception: delete_temp(self.config) if '*Failed_message' not in exception.args: raise exception if len(search_names) > 0: scrapper = Scrapper(self.config, self.output_columns_map) scrapper.run(search_names, message) input.add_status_column(scrapper.success_names) if not scrapper.forms_df.empty: output = Output(self.config) output.write(scrapper.forms_df) message.success_reply(output_file=output.path, no_processed=input.has_non_processed) return message.success_reply(output_file=None, no_processed=input.has_non_processed)
def execution(self): try: self.__bot_process() except Exception as ex: if '*Handled_error' not in ex.args: try: logger.error(self.config.get('error', 'unexpected'), exc_info=True) unexpected_message = Mail(self.config) unexpected_message.send_fail_to_admin(letter=4) except Exception: logger.error(self.config.get('error', 'unexpected'), exc_info=True) delete_temp(self.config)
def initialisation(): try: config = ConfigParser(interpolation=ExtendedInterpolation(), delimiters=(':')) config.read_file(open(path.join('settings', 'config.cfg'))) except Exception: logger.error('Config is not declared.') raise Exception logger.info(config.get('info', 'config_read')) if not os.path.exists(config.get('path', 'mapping')): logger.error( config.get('error', 'output_columns_path').format( config.get('path', 'mapping'))) raise Exception try: output_columns_map = get_output_map(config) except Exception: logger.error( config.get('error', 'output_columns').format(config.get('path', 'mapping'))) raise Exception delete_temp(config) return config, output_columns_map
output = Output(config) output.write(scrap_result['output_df']) message.success_reply(output_file=output.path, no_processed=input.has_non_processed) return message.success_reply(output_file=None, no_processed=input.has_non_processed) if __name__ == "__main__": make_loger() logger = logging.getLogger('mainFPDS') logger.info('FPDS bot execution started') try: config, output_columns_map = initialisation() main_steps(config, output_columns_map) delete_temp(config) except Exception as ex: if '*Handled_error' in ex.args: try: delete_temp(config) logger.error(config.get('error', 'unexpected'), exc_info=True) unexpected_message = Mail(config) unexpected_message.send_fail_to_admin(letter=4) except NameError: pass except Exception: logger.error(config.get('error', 'unexpected'), exc_info=True) delete_temp(config) logger.info('FPDS bot execution ended.')
def __init__(self): self.config = read_config(logger) self.output_columns_map = get_output_map(self.config, logger) delete_temp(self.config)