def main_steps(config, output_columns_map):
while True:
try:
message = Mail(config)
message.save_earlier_mail_attachment()
input = Input(config, file_path=message.saved_attachment)
search_names = input.parse_input(message)
break
except Exception as exception:
delete_temp(config)
if '*Failed_message' not in exception.args:
raise exception # unexpected error
if len(search_names) > 0:
logger.info(config.get('info', 'scrapping'))
try:
scrap_result = get_forms_df(config, search_names,
output_columns_map)
except Exception:
logger.error(config.get('error', 'fpds_site'))
message.send_fail_to_admin(letter=3)
raise Exception('*Handled_error')
input.add_status_column(scrap_result['status_res'])
if not scrap_result['output_df'].empty:
output = Output(config)
output.write(scrap_result['output_df'])
message.success_reply(output_file=output.path,
no_processed=input.has_non_processed)
return
message.success_reply(output_file=None,
no_processed=input.has_non_processed)
def __bot_process(self):
while True:
try:
message = Mail(self.config)
message.save_earlier_mail_attachment()
input = Input(self.config, file_path=message.saved_attachment)
search_names = input.parse_input(message)
break
except Exception as exception:
delete_temp(self.config)
if '*Failed_message' not in exception.args:
raise exception
if len(search_names) > 0:
scrapper = Scrapper(self.config, self.output_columns_map)
scrapper.run(search_names, message)
input.add_status_column(scrapper.success_names)
if not scrapper.forms_df.empty:
output = Output(self.config)
output.write(scrapper.forms_df)
message.success_reply(output_file=output.path,
no_processed=input.has_non_processed)
return
message.success_reply(output_file=None,
no_processed=input.has_non_processed)
def execution(self):
try:
self.__bot_process()
except Exception as ex:
if '*Handled_error' not in ex.args:
try:
logger.error(self.config.get('error', 'unexpected'),
exc_info=True)
unexpected_message = Mail(self.config)
unexpected_message.send_fail_to_admin(letter=4)
except Exception:
logger.error(self.config.get('error', 'unexpected'),
exc_info=True)
delete_temp(self.config)
def initialisation():
try:
config = ConfigParser(interpolation=ExtendedInterpolation(),
delimiters=(':'))
config.read_file(open(path.join('settings', 'config.cfg')))
except Exception:
logger.error('Config is not declared.')
raise Exception
logger.info(config.get('info', 'config_read'))
if not os.path.exists(config.get('path', 'mapping')):
logger.error(
config.get('error', 'output_columns_path').format(
config.get('path', 'mapping')))
raise Exception
try:
output_columns_map = get_output_map(config)
except Exception:
logger.error(
config.get('error',
'output_columns').format(config.get('path', 'mapping')))
raise Exception
delete_temp(config)
return config, output_columns_map
output = Output(config)
output.write(scrap_result['output_df'])
message.success_reply(output_file=output.path,
no_processed=input.has_non_processed)
return
message.success_reply(output_file=None,
no_processed=input.has_non_processed)
if __name__ == "__main__":
make_loger()
logger = logging.getLogger('mainFPDS')
logger.info('FPDS bot execution started')
try:
config, output_columns_map = initialisation()
main_steps(config, output_columns_map)
delete_temp(config)
except Exception as ex:
if '*Handled_error' in ex.args:
try:
delete_temp(config)
logger.error(config.get('error', 'unexpected'), exc_info=True)
unexpected_message = Mail(config)
unexpected_message.send_fail_to_admin(letter=4)
except NameError:
pass
except Exception:
logger.error(config.get('error', 'unexpected'), exc_info=True)
delete_temp(config)
logger.info('FPDS bot execution ended.')
def __init__(self):
self.config = read_config(logger)
self.output_columns_map = get_output_map(self.config, logger)
delete_temp(self.config)