def libor(ctx, start_date, end_date, config, skip_intermediate_panel_data, libor_csv_path,
intermediate_panel_path, skip_clean_last_dates, *args, **kwargs):
validate_dates(start_date, end_date)
start_date = start_date.date()
end_date = end_date.date()
try:
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(level=logging.WARNING)
config = read_config(file_path=config, command=ctx.command.name)
libor_file_path = validate_file_path(libor_csv_path, config, file_path_key='libor_file_path')
intermediate_panel_path = validate_file_path(intermediate_panel_path, config, file_path_key='intermediate_panel_path')
if os.path.isdir(libor_file_path):
click.echo('Error: el path ingresado para tasas libor es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo('Error: el path ingresado para el panel intermedio es un directorio')
exit()
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
ensure_dir_exists(os.path.split(libor_file_path)[0])
validate_url_config(config)
validate_url_has_value(config)
validate_libor_rates_config(config)
validate_libor_rates_has_values(config)
timeout = (
int(config.get('timeout'))
if 'timeout' in config.keys() else None
)
tries = int(config.get('tries', 1))
scraper = BCRALiborScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
rates=config.get('rates'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates
)
parsed = scraper.run(start_date, end_date)
processed_header = scraper.preprocess_header(scraper.rates)
write_file(processed_header, parsed, libor_file_path)
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Email().send_validation_group_email(execution_start_time, execution_end_time, start_date, end_date, skip_intermediate_panel_data, identifier='libor')
except InvalidConfigurationError as err:
click.echo(err)
def tce(ctx, config, start_date, end_date, refetch_start_date,
refetch_end_date, skip_intermediate_panel_data, dolar_csv_path,
euro_csv_path, intermediate_panel_path, skip_clean_last_dates):
try:
execution_start_hour = time.time()
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(format='%(message)s', level=logging.INFO)
logging.info(Figlet(font='standard').renderText('scraper tce'))
logging.info(f"Inicio de tiempo de ejecución: {execution_start_time}")
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
validate_entities_key_config(config)
validate_entities_key_has_values(config)
start_date = start_date.date()
end_date = end_date.date()
refetch_dates_range = []
if refetch_start_date and refetch_end_date:
validate_refetch_dates(start_date, end_date,
refetch_start_date.date(),
refetch_end_date.date())
refetch_dates_range = generate_dates_range(
refetch_start_date.date(), refetch_end_date.date())
elif refetch_start_date or refetch_end_date:
logging.warning(
'No se encontró fecha para refetch_start_date o refetch_end_date, no se hará refetch.'
)
dolar_file_path = validate_file_path(dolar_csv_path,
config,
file_path_key='dolar_file_path')
euro_file_path = validate_file_path(euro_csv_path,
config,
file_path_key='euro_file_path')
intermediate_panel_path = validate_file_path(
intermediate_panel_path,
config,
file_path_key='intermediate_panel_path')
if os.path.isdir(dolar_file_path):
click.echo('Error: el path ingresado para dolar es un directorio')
exit()
elif os.path.isdir(euro_file_path):
click.echo('Error: el path ingresado para euro es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo(
'Error: el path ingresado para el panel intermedio es un directorio'
)
exit()
ensure_dir_exists(os.path.split(dolar_file_path)[0])
ensure_dir_exists(os.path.split(euro_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (int(config.get('timeout'))
if 'timeout' in config.keys() else None)
tries = int(config.get('tries', 1))
scraper = BCRATCEScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
entities=config.get('entities'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates)
parsed = scraper.run(start_date, end_date, refetch_dates_range)
if parsed:
for coin in ['dolar', 'euro']:
csv_header = get_csv_header(coin, config)
if coin == 'dolar':
csv_name = dolar_file_path
else:
csv_name = euro_file_path
filtered_parsed = filter_parsed(parsed[coin], csv_header)
write_file(csv_header, filtered_parsed.values(), csv_name)
else:
click.echo("No se encontraron resultados")
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.info(f"Fin de tiempo de ejecución: {execution_end_time}")
execution_end_hour = time.time()
hours, rem = divmod(execution_end_hour - execution_start_hour, 3600)
minutes, seconds = divmod(rem, 60)
execution_total_time = "{:0>2}:{:0>2}:{:05.2f}".format(
int(hours), int(minutes), seconds)
Email().send_validation_group_email(execution_start_time,
execution_end_time,
execution_total_time,
start_date,
end_date,
skip_intermediate_panel_data,
identifier='tce')
except InvalidConfigurationError as err:
click.echo(err)
def exchange_rates(ctx, start_date, end_date, refetch_start_date,
refetch_end_date, config, skip_intermediate_panel_data,
tp_csv_path, tc_csv_path, intermediate_panel_path,
skip_clean_last_dates):
try:
execution_start_hour = time.time()
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(format='%(message)s', level=logging.INFO)
logging.info(Figlet(font='standard').renderText('scraper'))
logging.info(Figlet(font='standard').renderText('exchange rates'))
logging.info(f"Inicio de tiempo de ejecución: {execution_start_time}")
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
start_date = start_date.date()
end_date = end_date.date()
refetch_dates_range = []
if refetch_start_date and refetch_end_date:
validate_refetch_dates(start_date, end_date,
refetch_start_date.date(),
refetch_end_date.date())
refetch_dates_range = generate_dates_range(
refetch_start_date.date(), refetch_end_date.date())
elif refetch_start_date or refetch_end_date:
logging.warning(
'No se encontró fecha para refetch_start_date o refetch_end_date, no se hará refetch.'
)
tp_file_path = validate_file_path(tp_csv_path,
config,
file_path_key='tp_file_path')
tc_file_path = validate_file_path(tc_csv_path,
config,
file_path_key='tc_file_path')
intermediate_panel_path = validate_file_path(
intermediate_panel_path,
config,
file_path_key='intermediate_panel_path')
if os.path.isdir(tp_file_path):
click.echo(
'Error: el path ingresado para tipo de pase usd es un directorio'
)
exit()
elif os.path.isdir(tc_file_path):
click.echo(
'Error: el path ingresado para tipo de cambio local es un directorio'
)
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo(
'Error: el path ingresado para el panel intermedio es un directorio'
)
exit()
ensure_dir_exists(os.path.split(tp_file_path)[0])
ensure_dir_exists(os.path.split(tc_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (int(config.get('timeout'))
if 'timeout' in config.keys() else None)
tries = int(config.get('tries', 1))
scraper = BCRAExchangeRateScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates)
parsed = scraper.run(start_date, end_date, refetch_dates_range)
if parsed:
coins = config.get('coins')
csv_header = ['indice_tiempo']
csv_header.extend([v for v in coins.keys()])
write_file(csv_header, parsed['tp_usd'].values(), tp_file_path)
write_file(csv_header, parsed['tc_local'].values(), tc_file_path)
else:
click.echo("No se encontraron resultados")
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.info(f"Fin de tiempo de ejecución: {execution_end_time}")
execution_end_hour = time.time()
hours, rem = divmod(execution_end_hour - execution_start_hour, 3600)
minutes, seconds = divmod(rem, 60)
execution_total_time = "{:0>2}:{:0>2}:{:05.2f}".format(
int(hours), int(minutes), seconds)
Email().send_validation_group_email(execution_start_time,
execution_end_time,
execution_total_time,
start_date,
end_date,
skip_intermediate_panel_data,
identifier='exchange-rates')
except InvalidConfigurationError as err:
click.echo(err)
def libor(ctx, start_date, end_date, refetch_start_date, refetch_end_date,
config, skip_intermediate_panel_data, libor_csv_path,
intermediate_panel_path, skip_clean_last_dates, *args, **kwargs):
try:
execution_start_hour = time.time()
validate_dates(start_date, end_date)
start_date = start_date.date()
end_date = end_date.date()
refetch_dates_range = []
if refetch_start_date and refetch_end_date:
validate_refetch_dates(start_date, end_date,
refetch_start_date.date(),
refetch_end_date.date())
refetch_dates_range = generate_dates_range(
refetch_start_date.date(), refetch_end_date.date())
elif refetch_start_date or refetch_end_date:
logging.warning(
'No se encontró fecha para refetch_start_date o refetch_end_date, no se hará refetch.'
)
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(format='%(message)s', level=logging.INFO)
logging.info(Figlet(font='standard').renderText('scraper libor'))
logging.info(f"Inicio de tiempo de ejecución: {execution_start_time}")
config = read_config(file_path=config, command=ctx.command.name)
libor_file_path = validate_file_path(libor_csv_path,
config,
file_path_key='libor_file_path')
intermediate_panel_path = validate_file_path(
intermediate_panel_path,
config,
file_path_key='intermediate_panel_path')
if os.path.isdir(libor_file_path):
click.echo(
'Error: el path ingresado para tasas libor es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo(
'Error: el path ingresado para el panel intermedio es un directorio'
)
exit()
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
ensure_dir_exists(os.path.split(libor_file_path)[0])
validate_url_config(config)
validate_url_has_value(config)
validate_libor_rates_config(config)
validate_libor_rates_has_values(config)
timeout = (int(config.get('timeout'))
if 'timeout' in config.keys() else None)
tries = int(config.get('tries', 1))
scraper = BCRALiborScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
rates=config.get('rates'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates)
parsed = scraper.run(start_date, end_date, refetch_dates_range)
processed_header = scraper.preprocess_header(scraper.rates)
write_file(processed_header, parsed.values(), libor_file_path)
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.info(f"Fin de tiempo de ejecución: {execution_end_time}")
execution_end_hour = time.time()
hours, rem = divmod(execution_end_hour - execution_start_hour, 3600)
minutes, seconds = divmod(rem, 60)
execution_total_time = "{:0>2}:{:0>2}:{:05.2f}".format(
int(hours), int(minutes), seconds)
Email().send_validation_group_email(execution_start_time,
execution_end_time,
execution_total_time,
start_date,
end_date,
skip_intermediate_panel_data,
identifier='libor')
except InvalidConfigurationError as err:
click.echo(err)
def tce(ctx, config, start_date, end_date, skip_intermediate_panel_data, dolar_csv_path,
euro_csv_path, intermediate_panel_path, skip_clean_last_dates):
try:
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(level=logging.WARNING)
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
validate_entities_key_config(config)
validate_entities_key_has_values(config)
start_date = start_date.date()
end_date = end_date.date()
dolar_file_path = validate_file_path(dolar_csv_path, config, file_path_key='dolar_file_path')
euro_file_path = validate_file_path(euro_csv_path, config, file_path_key='euro_file_path')
intermediate_panel_path = validate_file_path(intermediate_panel_path, config, file_path_key='intermediate_panel_path')
if os.path.isdir(dolar_file_path):
click.echo('Error: el path ingresado para dolar es un directorio')
exit()
elif os.path.isdir(euro_file_path):
click.echo('Error: el path ingresado para euro es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo('Error: el path ingresado para el panel intermedio es un directorio')
exit()
ensure_dir_exists(os.path.split(dolar_file_path)[0])
ensure_dir_exists(os.path.split(euro_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (
int(config.get('timeout'))
if 'timeout' in config.keys() else None
)
tries = int(config.get('tries', 1))
scraper = BCRATCEScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
entities=config.get('entities'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates
)
parsed = scraper.run(start_date, end_date)
if parsed:
for coin in ['dolar', 'euro']:
csv_header = get_csv_header(coin, config)
if coin == 'dolar':
csv_name = dolar_file_path
else:
csv_name = euro_file_path
filtered_parsed = filter_parsed(parsed[coin], csv_header)
write_file(csv_header, filtered_parsed, csv_name)
else:
click.echo("No se encontraron resultados")
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Email().send_validation_group_email(execution_start_time, execution_end_time, start_date, end_date, skip_intermediate_panel_data, identifier='tce')
except InvalidConfigurationError as err:
click.echo(err)
def sml(ctx, config, start_date, end_date, skip_intermediate_panel_data, uruguayo_csv_path,
real_csv_path, intermediate_panel_path, skip_clean_last_dates):
try:
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(level=logging.WARNING)
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
start_date = start_date.date()
end_date = end_date.date()
peso_uruguayo_file_path = validate_file_path(uruguayo_csv_path, config, file_path_key='peso_uruguayo_file_path')
real_file_path = validate_file_path(real_csv_path, config, file_path_key='real_file_path')
intermediate_panel_path = validate_file_path(intermediate_panel_path, config, file_path_key='intermediate_panel_path')
if os.path.isdir(peso_uruguayo_file_path):
click.echo('Error: el path ingresado para peso uruguayo es un directorio')
exit()
elif os.path.isdir(real_file_path):
click.echo('Error: el path ingresado para real es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo('Error: el path ingresado para el panel intermedio es un directorio')
exit()
ensure_dir_exists(os.path.split(peso_uruguayo_file_path)[0])
ensure_dir_exists(os.path.split(real_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (
int(config.get('timeout'))
if 'timeout' in config.keys() else None
)
tries = int(config.get('tries', 1))
scraper = BCRASMLScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
types=config.get('types'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates
)
parsed = scraper.run(start_date, end_date)
if parsed:
for k in parsed.keys():
if k == 'peso_uruguayo':
csv_header = ['indice_tiempo']
csv_header.extend(config['types']['peso_uruguayo'].values())
write_file(csv_header, parsed['peso_uruguayo'], peso_uruguayo_file_path)
elif k == 'real':
csv_header = ['indice_tiempo']
csv_header.extend(config['types']['real'].values())
write_file(csv_header, parsed['real'], real_file_path)
else:
click.echo("No se encontraron resultados")
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Email().send_validation_group_email(execution_start_time, execution_end_time, start_date, end_date, skip_intermediate_panel_data, identifier='sml')
except InvalidConfigurationError as err:
click.echo(err)
def exchange_rates(ctx, start_date, end_date, config, skip_intermediate_panel_data,
tp_csv_path, tc_csv_path, intermediate_panel_path, skip_clean_last_dates):
try:
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(level=logging.WARNING)
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
start_date = start_date.date()
end_date = end_date.date()
tp_file_path = validate_file_path(tp_csv_path, config, file_path_key='tp_file_path')
tc_file_path = validate_file_path(tc_csv_path, config, file_path_key='tc_file_path')
intermediate_panel_path = validate_file_path(intermediate_panel_path, config, file_path_key='intermediate_panel_path')
if os.path.isdir(tp_file_path):
click.echo('Error: el path ingresado para tipo de pase usd es un directorio')
exit()
elif os.path.isdir(tc_file_path):
click.echo('Error: el path ingresado para tipo de cambio local es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo('Error: el path ingresado para el panel intermedio es un directorio')
exit()
ensure_dir_exists(os.path.split(tp_file_path)[0])
ensure_dir_exists(os.path.split(tc_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (
int(config.get('timeout'))
if 'timeout' in config.keys() else None
)
tries = int(config.get('tries', 1))
scraper = BCRAExchangeRateScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates
)
parsed = scraper.run(start_date, end_date)
if parsed:
coins = config.get('coins')
csv_header = ['indice_tiempo']
csv_header.extend([v for v in coins.keys()])
write_file(csv_header, parsed['tp_usd'], tp_file_path)
write_file(csv_header, parsed['tc_local'], tc_file_path)
else:
click.echo("No se encontraron resultados")
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Email().send_validation_group_email(execution_start_time, execution_end_time, start_date, end_date, skip_intermediate_panel_data, identifier='exchange-rates')
except InvalidConfigurationError as err:
click.echo(err)