def exchange_rates(ctx, start_date, end_date, refetch_start_date, refetch_end_date, config, skip_intermediate_panel_data, tp_csv_path, tc_csv_path, intermediate_panel_path, skip_clean_last_dates): try: execution_start_hour = time.time() execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") logging.basicConfig(format='%(message)s', level=logging.INFO) logging.info(Figlet(font='standard').renderText('scraper')) logging.info(Figlet(font='standard').renderText('exchange rates')) logging.info(f"Inicio de tiempo de ejecución: {execution_start_time}") config = read_config(file_path=config, command=ctx.command.name) validate_url_config(config) validate_url_has_value(config) validate_coins_key_config(config) validate_coins_key_has_values(config) validate_dates(start_date, end_date) start_date = start_date.date() end_date = end_date.date() refetch_dates_range = [] if refetch_start_date and refetch_end_date: validate_refetch_dates(start_date, end_date, refetch_start_date.date(), refetch_end_date.date()) refetch_dates_range = generate_dates_range( refetch_start_date.date(), refetch_end_date.date()) elif refetch_start_date or refetch_end_date: logging.warning( 'No se encontró fecha para refetch_start_date o refetch_end_date, no se hará refetch.' ) tp_file_path = validate_file_path(tp_csv_path, config, file_path_key='tp_file_path') tc_file_path = validate_file_path(tc_csv_path, config, file_path_key='tc_file_path') intermediate_panel_path = validate_file_path( intermediate_panel_path, config, file_path_key='intermediate_panel_path') if os.path.isdir(tp_file_path): click.echo( 'Error: el path ingresado para tipo de pase usd es un directorio' ) exit() elif os.path.isdir(tc_file_path): click.echo( 'Error: el path ingresado para tipo de cambio local es un directorio' ) exit() elif os.path.isdir(intermediate_panel_path): click.echo( 'Error: el path ingresado para el panel intermedio es un directorio' ) exit() ensure_dir_exists(os.path.split(tp_file_path)[0]) ensure_dir_exists(os.path.split(tc_file_path)[0]) ensure_dir_exists(os.path.split(intermediate_panel_path)[0]) timeout = (int(config.get('timeout')) if 'timeout' in config.keys() else None) tries = int(config.get('tries', 1)) scraper = BCRAExchangeRateScraper( url=config.get('url'), timeout=timeout, tries=tries, coins=config.get('coins'), skip_intermediate_panel_data=skip_intermediate_panel_data, intermediate_panel_path=intermediate_panel_path, skip_clean_last_dates=skip_clean_last_dates) parsed = scraper.run(start_date, end_date, refetch_dates_range) if parsed: coins = config.get('coins') csv_header = ['indice_tiempo'] csv_header.extend([v for v in coins.keys()]) write_file(csv_header, parsed['tp_usd'].values(), tp_file_path) write_file(csv_header, parsed['tc_local'].values(), tc_file_path) else: click.echo("No se encontraron resultados") execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") logging.info(f"Fin de tiempo de ejecución: {execution_end_time}") execution_end_hour = time.time() hours, rem = divmod(execution_end_hour - execution_start_hour, 3600) minutes, seconds = divmod(rem, 60) execution_total_time = "{:0>2}:{:0>2}:{:05.2f}".format( int(hours), int(minutes), seconds) Email().send_validation_group_email(execution_start_time, execution_end_time, execution_total_time, start_date, end_date, skip_intermediate_panel_data, identifier='exchange-rates') except InvalidConfigurationError as err: click.echo(err)
def exchange_rates(ctx, start_date, end_date, config, use_intermediate_panel, tp_csv_path, tc_csv_path, intermediate_panel_path): try: logging.basicConfig(level=logging.WARNING) config = read_config(file_path=config, command=ctx.command.name) validate_url_config(config) validate_url_has_value(config) validate_coins_key_config(config) validate_coins_key_has_values(config) validate_dates(start_date, end_date) tp_file_path = validate_file_path(tp_csv_path, config, file_path_key='tp_file_path') tc_file_path = validate_file_path(tc_csv_path, config, file_path_key='tc_file_path') intermediate_panel_path = validate_file_path( intermediate_panel_path, config, file_path_key='intermediate_panel_path') if os.path.isdir(tp_file_path): click.echo( 'Error: el path ingresado para tipo de pase usd es un directorio' ) exit() elif os.path.isdir(tc_file_path): click.echo( 'Error: el path ingresado para tipo de cambio local es un directorio' ) exit() elif os.path.isdir(intermediate_panel_path): click.echo( 'Error: el path ingresado para el panel intermedio es un directorio' ) exit() ensure_dir_exists(os.path.split(tp_file_path)[0]) ensure_dir_exists(os.path.split(tc_file_path)[0]) ensure_dir_exists(os.path.split(intermediate_panel_path)[0]) timeout = (int(config.get('timeout')) if 'timeout' in config.keys() else None) tries = int(config.get('tries', 1)) scraper = BCRAExchangeRateScraper( url=config.get('url'), timeout=timeout, tries=tries, coins=config.get('coins'), use_intermediate_panel=use_intermediate_panel, intermediate_panel_path=intermediate_panel_path) parsed = scraper.run(start_date, end_date) if parsed: coins = config.get('coins') csv_header = ['indice_tiempo'] csv_header.extend([v for v in coins.keys()]) parsed['tp_usd'].reverse() parsed['tc_local'].reverse() write_file(csv_header, parsed['tp_usd'], tp_file_path) write_file(csv_header, parsed['tc_local'], tc_file_path) else: click.echo("No se encontraron resultados") except InvalidConfigurationError as err: click.echo(err)
def exchange_rates(ctx, start_date, end_date, config, skip_intermediate_panel_data, tp_csv_path, tc_csv_path, intermediate_panel_path, skip_clean_last_dates): try: execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") logging.basicConfig(level=logging.WARNING) config = read_config(file_path=config, command=ctx.command.name) validate_url_config(config) validate_url_has_value(config) validate_coins_key_config(config) validate_coins_key_has_values(config) validate_dates(start_date, end_date) start_date = start_date.date() end_date = end_date.date() tp_file_path = validate_file_path(tp_csv_path, config, file_path_key='tp_file_path') tc_file_path = validate_file_path(tc_csv_path, config, file_path_key='tc_file_path') intermediate_panel_path = validate_file_path(intermediate_panel_path, config, file_path_key='intermediate_panel_path') if os.path.isdir(tp_file_path): click.echo('Error: el path ingresado para tipo de pase usd es un directorio') exit() elif os.path.isdir(tc_file_path): click.echo('Error: el path ingresado para tipo de cambio local es un directorio') exit() elif os.path.isdir(intermediate_panel_path): click.echo('Error: el path ingresado para el panel intermedio es un directorio') exit() ensure_dir_exists(os.path.split(tp_file_path)[0]) ensure_dir_exists(os.path.split(tc_file_path)[0]) ensure_dir_exists(os.path.split(intermediate_panel_path)[0]) timeout = ( int(config.get('timeout')) if 'timeout' in config.keys() else None ) tries = int(config.get('tries', 1)) scraper = BCRAExchangeRateScraper( url=config.get('url'), timeout=timeout, tries=tries, coins=config.get('coins'), skip_intermediate_panel_data=skip_intermediate_panel_data, intermediate_panel_path=intermediate_panel_path, skip_clean_last_dates=skip_clean_last_dates ) parsed = scraper.run(start_date, end_date) if parsed: coins = config.get('coins') csv_header = ['indice_tiempo'] csv_header.extend([v for v in coins.keys()]) write_file(csv_header, parsed['tp_usd'], tp_file_path) write_file(csv_header, parsed['tc_local'], tc_file_path) else: click.echo("No se encontraron resultados") execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") Email().send_validation_group_email(execution_start_time, execution_end_time, start_date, end_date, skip_intermediate_panel_data, identifier='exchange-rates') except InvalidConfigurationError as err: click.echo(err)