def test_html_is_valid(self):
"""Probar que el html sea valido"""
url = ""
single_date = date(2019, 3, 4)
coins = {}
with patch.object(BCRASMLScraper,
'fetch_content',
return_value='''
<table class="table table-BCRA table-bordered table-hover
table-responsive">
<thead>
</thead>
<tbody>
</tbody>
</table>
'''):
scraper = BCRASMLScraper(url, coins, False)
content = scraper.fetch_content(single_date)
soup = BeautifulSoup(content, "html.parser")
table = soup.find('table')
head = table.find('thead') if table else None
body = table.find('tbody') if table else None
assert table is not None
assert head is not None
assert body is not None
def test_parse_content_not_table(self):
start_date = datetime(2019, 4, 22)
end_date = datetime(2019, 4, 22)
url = ''
coins = {}
contents = {'peso_uruguayo': '', 'real': ''}
scraper = BCRASMLScraper(url, coins, False)
result = scraper.parse_contents(contents, start_date, end_date)
assert result == {'peso_uruguayo': [], 'real': []}
def test_get_intermediate_panel_data_from_empty_parsed(self):
url = ''
parsed = {}
coins = {'peso_uruguayo': 'Peso Uruguayo', 'real': 'Real'}
scraper = BCRASMLScraper(url, coins, True)
result = scraper.get_intermediate_panel_data_from_parsed(parsed)
assert result == []
def test_parse_contents(self):
url = ''
contents = {'peso_uruguayo': 'foo', 'real': 'foo'}
start_date = datetime(2019, 4, 24)
end_date = datetime(2019, 4, 24)
coins = {"peso_uruguayo": "Peso Uruguayo", "real": "Real"}
content_peso = [{
'coin': 'peso_uruguayo',
'indice_tiempo': '24/04/2019',
'Tipo de cambio de Referencia': '43,47830',
'Tipo de cambio URINUSCA': '34,51000',
'Tipo de cambio SML Peso Uruguayo': '1,25990',
'Tipo de cambio SML Uruguayo Peso': '0,79375'
}]
content_real = [{
'coin': 'real',
'indice_tiempo': '24/04/2019',
'Tipo de cambio de Referencia': '43,47830',
'Tipo de cambio PTAX': '3,96270',
'Tipo de cambio SML Peso Real': '10,97190',
'Tipo de cambio SML Real Peso': '0,09115'
}]
with patch.object(BCRASMLScraper,
'parse_content',
side_effect=[content_peso, content_real]):
scraper = BCRASMLScraper(url, coins, False)
result = scraper.parse_contents(contents, start_date, end_date)
assert result == {
'peso_uruguayo': [{
'Tipo de cambio de Referencia': '43,47830',
'Tipo de cambio URINUSCA': '34,51000',
'Tipo de cambio SML Peso Uruguayo': '1,25990',
'Tipo de cambio SML Uruguayo Peso': '0,79375',
'indice_tiempo': '24/04/2019'
}],
'real': [{
'Tipo de cambio de Referencia': '43,47830',
'Tipo de cambio PTAX': '3,96270',
'Tipo de cambio SML Peso Real': '10,97190',
'Tipo de cambio SML Real Peso': '0,09115',
'indice_tiempo': '24/04/2019'
}]
}
def test_validate_coin_in_configuration_file_false(self):
coins = {}
url = 'foo.com'
coin = "Rel"
options = []
for option_text in ['Seleccione moneda', 'Real', 'Peso Uruguayo']:
mock = MagicMock()
mock.text = option_text
options.append(mock)
scraper = BCRASMLScraper(url, coins, intermediate_panel_path=None, use_intermediate_panel=False)
coin_in_configuration_file = scraper.validate_coin_in_configuration_file(coin, options)
assert coin_in_configuration_file is False
def test_validate_coin_in_configuration_file_true(self):
coins = {"real": "Real", "peso_uruguayo": "Peso Uruguayo"}
url = 'foo.com'
coin = "Real"
options = []
for option_text in ['Seleccione moneda', 'Real', 'Peso Uruguayo']:
mock = MagicMock()
mock.text = option_text
options.append(mock)
scraper = BCRASMLScraper(url, coins, False)
coin_in_configuration_file = scraper.validate_coin_in_configuration_file(
coin, options)
assert coin_in_configuration_file is True
def test_fetch_contents(self):
start_date = datetime(2019, 4, 11)
end_date = datetime(2019, 4, 11)
url = ''
coins = {"peso_uruguayo": "Peso Uruguayo", "real": "Real"}
content = 'foo'
with patch.object(BCRASMLScraper,
'fetch_content',
return_value=content):
scraper = BCRASMLScraper(url, coins, False)
result = scraper.fetch_contents(start_date, end_date)
assert result == {'peso_uruguayo': 'foo', 'real': 'foo'}
def test_parse_from_intermediate_panel_empty_uruguayo(self):
"""Probar parseo desde el archivo intermedio"""
start_date = '2019-03-06'
end_date = '2019-03-06'
coins = {"peso_uruguayo": "Peso Uruguayo", "real": "Real"}
url = ''
intermediate_panel_df = MagicMock()
intermediate_panel_df = {
'indice_tiempo':
['2019-03-06', '2019-03-06', '2019-03-06', '2019-03-06'],
'coin': [
'real',
'real',
'real',
'real',
],
'type': [
'Tipo de cambio de Referencia', 'Tipo de cambio PTAX',
'Tipo de cambio SML Peso Real', 'Tipo de cambio SML Real Peso'
],
'value': ['40.48170', '3.83000', '10.56965', '0.09465']
}
with patch.object(
BCRASMLScraper,
'read_intermediate_panel_dataframe',
return_value=pd.DataFrame(data=intermediate_panel_df)):
scraper = BCRASMLScraper(url, coins, True)
content = scraper.parse_from_intermediate_panel(
start_date,
end_date,
)
assert content == {
'peso_uruguayo': [],
'real': [{
'indice_tiempo': '2019-03-06',
'Tipo de cambio de Referencia': '40.48170',
'Tipo de cambio PTAX': '3.83000',
'Tipo de cambio SML Peso Real': '10.56965',
'Tipo de cambio SML Real Peso': '0.09465'
}]
}
def test_fetch_content_invalid_url_patching_driver(self):
"""Probar fetch content con url invalida"""
coins = {}
url = ''
mocked_driver = MagicMock()
mocked_driver.page_source = 400
with patch.object(BCRASMLScraper,
'get_browser_driver',
return_value=mocked_driver):
with patch.object(BCRASMLScraper,
'validate_coin_in_configuration_file',
return_value=True):
scraper = BCRASMLScraper(url, coins, False)
content = scraper.fetch_content(coins)
assert content == 400
def test_parse_content_with_valid_content(self):
start_date = datetime(2019, 4, 24)
end_date = datetime(2019, 4, 24)
coin = "peso_uruguayo"
content = '''
<table colspan="3" class="table table-BCRA table-bordered
table-hover table-responsive">
<thead>
<tr>
<th>Fecha</th>
<th>Tipo de cambio de Referencia</th>
<th>Tipo de cambio URINUSCA</th>
<th>Tipo de cambio SML Peso Uruguayo</th>
<th>Tipo de cambio SML Uruguayo Peso</th>
</tr>
</thead>
<tbody>
<tr>
<td>24/04/2019</td>
<td>43,47830</td>
<td>34,51000</td>
<td>1,25990</td>
<td>0,79375</td>
</tr>
</tbody>
</table>
'''
scraper = BCRASMLScraper(False, coin, intermediate_panel_path=None, use_intermediate_panel=False)
result = scraper.parse_content(
content, coin, start_date, end_date
)
assert result == [
{
'coin': 'peso_uruguayo',
'indice_tiempo': '24/04/2019',
'Tipo de cambio de Referencia': '43,47830',
'Tipo de cambio URINUSCA': '34,51000',
'Tipo de cambio SML Peso Uruguayo': '1,25990',
'Tipo de cambio SML Uruguayo Peso': '0,79375'
}
]
def test_html_is_not_valid(self):
"""Probar que el html no sea valido"""
url = ""
single_date = date(2019, 3, 4)
coins = {}
with patch.object(BCRASMLScraper, 'fetch_content', return_value=' '):
scraper = BCRASMLScraper(url, coins, False)
content = scraper.fetch_content(single_date)
soup = BeautifulSoup(content, "html.parser")
table = soup.find('table')
head = table.find('thead') if table else None
body = table.find('tbody') if table else None
assert table is None
assert head is None
assert body is None
def test_parse_content_with_non_valid_content(self):
start_date = datetime(2019, 4, 11)
end_date = datetime(2019, 4, 11)
coin = "peso_uruguayo"
content = '''
<table colspan="3" class="table table-BCRA table-bordered
table-hover table-responsive">
<thead>
</thead>
<tbody>
</tbody>
</table>
'''
scraper = BCRASMLScraper(False, coin, False)
result = scraper.parse_content(content, coin, start_date, end_date)
assert result == []
def test_parse_content_not_head(self):
start_date = datetime(2019, 4, 22)
end_date = datetime(2019, 4, 22)
url = ''
coins = {}
contents = {
'peso_uruguayo':
'''
<table class="table table-BCRA table-bordered table-hover\
table-responsive" colspan="3">
</table>''',
'real':
'''
<table class="table table-BCRA table-bordered table-hover\
table-responsive" colspan="3">
</table>'''
}
scraper = BCRASMLScraper(url, coins, False)
result = scraper.parse_contents(contents, start_date, end_date)
assert result == {'peso_uruguayo': [], 'real': []}
def test_fetch_content_patching_driver(self):
"""Probar fetch content"""
coins = {}
url = ''
mocked_driver = MagicMock()
mocked_driver.page_source = "foo"
mocked_driver.status_code = 200
with patch.object(
BCRASMLScraper,
'get_browser_driver',
return_value=mocked_driver
):
with patch.object(
BCRASMLScraper,
'validate_coin_in_configuration_file',
return_value=True
):
scraper = BCRASMLScraper(url, coins, intermediate_panel_path=None, use_intermediate_panel=False)
content = scraper.fetch_content(coins)
assert content == "foo"
def test_preprocessed_rows(self):
rows = [
{
'indice_tiempo': '2019-03-06',
'Tipo de cambio de Referencia': '40.48170',
'Tipo de cambio URINUSCA': '32.68200',
'Tipo de cambio SML Peso Uruguayo': '1.23865',
'Tipo de cambio SML Uruguayo Peso': '0.80735'
},
{
'indice_tiempo': '2019-03-06',
'Tipo de cambio de Referencia': '40.48170',
'Tipo de cambio PTAX': '3.83000',
'Tipo de cambio SML Peso Real': '10.56965',
'Tipo de cambio SML Real Peso': '0.09465'
}
]
scraper = BCRASMLScraper(False, rows, intermediate_panel_path=None, use_intermediate_panel=False)
result = scraper.preprocess_rows(rows)
assert result == [
{
'indice_tiempo': date(2019, 3, 6),
'Tipo de cambio de Referencia': Decimal('40.48170'),
'Tipo de cambio URINUSCA': Decimal('32.68200'),
'Tipo de cambio SML Peso Uruguayo': Decimal('1.23865'),
'Tipo de cambio SML Uruguayo Peso': Decimal('0.80735')
},
{
'indice_tiempo': date(2019, 3, 6),
'Tipo de cambio de Referencia': Decimal('40.48170'),
'Tipo de cambio PTAX': Decimal('3.83000'),
'Tipo de cambio SML Peso Real': Decimal('10.56965'),
'Tipo de cambio SML Real Peso': Decimal('0.09465')
}
]
def test_run_using_intermediate_panel(self):
start_date = datetime(2019, 5, 6)
end_date = datetime(2019, 5, 6)
url = '''
http://www.bcra.gov.ar/PublicacionesEstadisticas/Tipo_de_cambio_sml.asp
'''
coins = {"peso_uruguayo": "Peso Uruguayo", "real": "Real"}
parsed = {
'peso_uruguayo': [{
'Tipo de cambio de Referencia':
Decimal('44.89670'),
'Tipo de cambio URINUSCA':
Decimal('35.03600'),
'Tipo de cambio SML Peso Uruguayo':
Decimal('1.28145'),
'Tipo de cambio SML Uruguayo Peso':
Decimal('0.78040'),
'indice_tiempo':
date(2019, 5, 6)
}],
'real': [{
'Tipo de cambio de Referencia': Decimal('44.89670'),
'Tipo de cambio PTAX': Decimal('3.96210'),
'Tipo de cambio SML Peso Real': Decimal('11.33155'),
'Tipo de cambio SML Real Peso': Decimal('0.08825'),
'indice_tiempo': date(2019, 5, 6)
}]
}
peso_uruguayo_preprocess = [{
'Tipo de cambio de Referencia':
Decimal('44.89670'),
'Tipo de cambio URINUSCA':
Decimal('35.03600'),
'Tipo de cambio SML Peso Uruguayo':
Decimal('1.28145'),
'Tipo de cambio SML Uruguayo Peso':
Decimal('0.78040'),
'indice_tiempo':
date(2019, 5, 6)
}]
real_preprocess = [{
'Tipo de cambio de Referencia': Decimal('44.89670'),
'Tipo de cambio PTAX': Decimal('3.96210'),
'Tipo de cambio SML Peso Real': Decimal('11.33155'),
'Tipo de cambio SML Real Peso': Decimal('0.08825'),
'indice_tiempo': date(2019, 5, 6)
}]
with patch.object(BCRASMLScraper,
'parse_from_intermediate_panel',
return_value=parsed):
with patch.object(
BCRASMLScraper,
'preprocess_rows',
side_effect=[peso_uruguayo_preprocess, real_preprocess]):
scraper = BCRASMLScraper(url, coins, True)
result = scraper.run(start_date, end_date)
assert result == {
'peso_uruguayo': [{
'Tipo de cambio de Referencia':
Decimal('44.89670'),
'Tipo de cambio URINUSCA':
Decimal('35.03600'),
'Tipo de cambio SML Peso Uruguayo':
Decimal('1.28145'),
'Tipo de cambio SML Uruguayo Peso':
Decimal('0.78040'),
'indice_tiempo':
date(2019, 5, 6)
}],
'real': [{
'Tipo de cambio de Referencia':
Decimal('44.89670'),
'Tipo de cambio PTAX':
Decimal('3.96210'),
'Tipo de cambio SML Peso Real':
Decimal('11.33155'),
'Tipo de cambio SML Real Peso':
Decimal('0.08825'),
'indice_tiempo':
date(2019, 5, 6)
}]
}
def test_parse_from_intermediate_panel_empty_real(self):
"""Probar parseo desde el archivo intermedio"""
start_date = '2019-03-06'
end_date = '2019-03-06'
coins = {
"peso_uruguayo": "Peso Uruguayo",
"real": "Real"
}
url = ''
intermediate_panel_df = MagicMock()
intermediate_panel_df = {
'indice_tiempo': [
'2019-03-06',
'2019-03-06',
'2019-03-06',
'2019-03-06'
],
'coin': [
'peso_uruguayo',
'peso_uruguayo',
'peso_uruguayo',
'peso_uruguayo'
],
'type': [
'Tipo de cambio de Referencia',
'Tipo de cambio URINUSCA',
'Tipo de cambio SML Peso Uruguayo',
'Tipo de cambio SML Uruguayo Peso'
],
'value': [
'40.48170',
'32.68200',
'1.23865',
'0.80735'
]
}
with patch.object(
BCRASMLScraper,
'read_intermediate_panel_dataframe',
return_value=pd.DataFrame(data=intermediate_panel_df)
):
scraper = BCRASMLScraper(url, coins, intermediate_panel_path=None, use_intermediate_panel=True)
content = scraper.parse_from_intermediate_panel(
start_date, end_date,
)
assert content == {
'peso_uruguayo': [
{
'indice_tiempo': '2019-03-06',
'Tipo de cambio de Referencia': '40.48170',
'Tipo de cambio URINUSCA': '32.68200',
'Tipo de cambio SML Peso Uruguayo': '1.23865',
'Tipo de cambio SML Uruguayo Peso': '0.80735'
}
],
'real': []
}
def test_get_intermediate_panel_data_from_parsed(self):
parsed = {
'peso_uruguayo': [{
'Tipo de cambio de Referencia':
Decimal('44.89670'),
'Tipo de cambio URINUSCA':
Decimal('35.03600'),
'Tipo de cambio SML Peso Uruguayo':
Decimal('1.28145'),
'Tipo de cambio SML Uruguayo Peso':
Decimal('0.78040'),
'indice_tiempo':
date(2019, 5, 6)
}],
'real': [{
'Tipo de cambio de Referencia': Decimal('44.89670'),
'Tipo de cambio PTAX': Decimal('3.96210'),
'Tipo de cambio SML Peso Real': Decimal('11.33155'),
'Tipo de cambio SML Real Peso': Decimal('0.08825'),
'indice_tiempo': date(2019, 5, 6)
}]
}
url = ''
coins = {'peso_uruguayo': 'Peso Uruguayo', 'real': 'Real'}
scraper = BCRASMLScraper(url, coins, True)
result = scraper.get_intermediate_panel_data_from_parsed(parsed)
assert result == [{
'indice_tiempo': date(2019, 5, 6),
'coin': 'peso_uruguayo',
'type': 'Tipo de cambio de Referencia',
'value': Decimal('44.89670')
}, {
'indice_tiempo': date(2019, 5, 6),
'coin': 'peso_uruguayo',
'type': 'Tipo de cambio URINUSCA',
'value': Decimal('35.03600')
}, {
'indice_tiempo': date(2019, 5, 6),
'coin': 'peso_uruguayo',
'type': 'Tipo de cambio SML Peso Uruguayo',
'value': Decimal('1.28145')
}, {
'indice_tiempo': date(2019, 5, 6),
'coin': 'peso_uruguayo',
'type': 'Tipo de cambio SML Uruguayo Peso',
'value': Decimal('0.78040')
}, {
'indice_tiempo': date(2019, 5, 6),
'coin': 'real',
'type': 'Tipo de cambio de Referencia',
'value': Decimal('44.89670')
}, {
'indice_tiempo': date(2019, 5, 6),
'coin': 'real',
'type': 'Tipo de cambio PTAX',
'value': Decimal('3.96210')
}, {
'indice_tiempo': date(2019, 5, 6),
'coin': 'real',
'type': 'Tipo de cambio SML Peso Real',
'value': Decimal('11.33155')
}, {
'indice_tiempo': date(2019, 5, 6),
'coin': 'real',
'type': 'Tipo de cambio SML Real Peso',
'value': Decimal('0.08825')
}]
def sml(ctx, config, start_date, end_date, refetch_start_date,
refetch_end_date, skip_intermediate_panel_data, uruguayo_csv_path,
real_csv_path, intermediate_panel_path, skip_clean_last_dates):
try:
execution_start_hour = time.time()
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(format='%(message)s', level=logging.INFO)
logging.info(Figlet(font='standard').renderText('scraper sml'))
logging.info(f"Inicio de tiempo de ejecución: {execution_start_time}")
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
start_date = start_date.date()
end_date = end_date.date()
refetch_dates_range = []
if refetch_start_date and refetch_end_date:
validate_refetch_dates(start_date, end_date,
refetch_start_date.date(),
refetch_end_date.date())
refetch_dates_range = generate_dates_range(
refetch_start_date.date(), refetch_end_date.date())
elif refetch_start_date or refetch_end_date:
logging.warning(
'No se encontró fecha para refetch_start_date o refetch_end_date, no se hará refetch.'
)
peso_uruguayo_file_path = validate_file_path(
uruguayo_csv_path, config, file_path_key='peso_uruguayo_file_path')
real_file_path = validate_file_path(real_csv_path,
config,
file_path_key='real_file_path')
intermediate_panel_path = validate_file_path(
intermediate_panel_path,
config,
file_path_key='intermediate_panel_path')
if os.path.isdir(peso_uruguayo_file_path):
click.echo(
'Error: el path ingresado para peso uruguayo es un directorio')
exit()
elif os.path.isdir(real_file_path):
click.echo('Error: el path ingresado para real es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo(
'Error: el path ingresado para el panel intermedio es un directorio'
)
exit()
ensure_dir_exists(os.path.split(peso_uruguayo_file_path)[0])
ensure_dir_exists(os.path.split(real_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (int(config.get('timeout'))
if 'timeout' in config.keys() else None)
tries = int(config.get('tries', 1))
scraper = BCRASMLScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
types=config.get('types'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates)
parsed = scraper.run(start_date, end_date, refetch_dates_range)
if parsed:
for k in parsed.keys():
if k == 'peso_uruguayo':
csv_header = ['indice_tiempo']
csv_header.extend(
config['types']['peso_uruguayo'].values())
write_file(csv_header, parsed['peso_uruguayo'].values(),
peso_uruguayo_file_path)
elif k == 'real':
csv_header = ['indice_tiempo']
csv_header.extend(config['types']['real'].values())
write_file(csv_header, parsed['real'].values(),
real_file_path)
else:
click.echo("No se encontraron resultados")
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.info(f"Fin de tiempo de ejecución: {execution_end_time}")
execution_end_hour = time.time()
hours, rem = divmod(execution_end_hour - execution_start_hour, 3600)
minutes, seconds = divmod(rem, 60)
execution_total_time = "{:0>2}:{:0>2}:{:05.2f}".format(
int(hours), int(minutes), seconds)
Email().send_validation_group_email(execution_start_time,
execution_end_time,
execution_total_time,
start_date,
end_date,
skip_intermediate_panel_data,
identifier='sml')
except InvalidConfigurationError as err:
click.echo(err)
def sml(ctx, config, start_date, end_date, use_intermediate_panel,
uruguayo_csv_path, real_csv_path, intermediate_panel_path):
try:
logging.basicConfig(level=logging.WARNING)
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
peso_uruguayo_file_path = validate_file_path(
uruguayo_csv_path, config, file_path_key='peso_uruguayo_file_path')
real_file_path = validate_file_path(real_csv_path,
config,
file_path_key='real_file_path')
intermediate_panel_path = validate_file_path(
intermediate_panel_path,
config,
file_path_key='intermediate_panel_path')
if os.path.isdir(peso_uruguayo_file_path):
click.echo(
'Error: el path ingresado para peso uruguayo es un directorio')
exit()
elif os.path.isdir(real_file_path):
click.echo('Error: el path ingresado para real es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo(
'Error: el path ingresado para el panel intermedio es un directorio'
)
exit()
ensure_dir_exists(os.path.split(peso_uruguayo_file_path)[0])
ensure_dir_exists(os.path.split(real_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (int(config.get('timeout'))
if 'timeout' in config.keys() else None)
tries = int(config.get('tries', 1))
scraper = BCRASMLScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
use_intermediate_panel=use_intermediate_panel,
intermediate_panel_path=intermediate_panel_path)
parsed = scraper.run(start_date, end_date)
if parsed:
for k, v in parsed.items():
if k == 'peso_uruguayo':
csv_header = [
'indice_tiempo', 'Tipo de cambio de Referencia',
'Tipo de cambio URINUSCA',
'Tipo de cambio SML Peso Uruguayo',
'Tipo de cambio SML Uruguayo Peso'
]
file_path = peso_uruguayo_file_path
elif k == 'real':
csv_header = [
'indice_tiempo', 'Tipo de cambio de Referencia',
'Tipo de cambio PTAX', 'Tipo de cambio SML Peso Real',
'Tipo de cambio SML Real Peso'
]
file_path = real_file_path
parsed[k].reverse()
write_file(csv_header, parsed[k], file_path)
else:
click.echo("No se encontraron resultados")
except InvalidConfigurationError as err:
click.echo(err)
def sml(ctx, config, start_date, end_date, skip_intermediate_panel_data, uruguayo_csv_path,
real_csv_path, intermediate_panel_path, skip_clean_last_dates):
try:
execution_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logging.basicConfig(level=logging.WARNING)
config = read_config(file_path=config, command=ctx.command.name)
validate_url_config(config)
validate_url_has_value(config)
validate_coins_key_config(config)
validate_coins_key_has_values(config)
validate_dates(start_date, end_date)
start_date = start_date.date()
end_date = end_date.date()
peso_uruguayo_file_path = validate_file_path(uruguayo_csv_path, config, file_path_key='peso_uruguayo_file_path')
real_file_path = validate_file_path(real_csv_path, config, file_path_key='real_file_path')
intermediate_panel_path = validate_file_path(intermediate_panel_path, config, file_path_key='intermediate_panel_path')
if os.path.isdir(peso_uruguayo_file_path):
click.echo('Error: el path ingresado para peso uruguayo es un directorio')
exit()
elif os.path.isdir(real_file_path):
click.echo('Error: el path ingresado para real es un directorio')
exit()
elif os.path.isdir(intermediate_panel_path):
click.echo('Error: el path ingresado para el panel intermedio es un directorio')
exit()
ensure_dir_exists(os.path.split(peso_uruguayo_file_path)[0])
ensure_dir_exists(os.path.split(real_file_path)[0])
ensure_dir_exists(os.path.split(intermediate_panel_path)[0])
timeout = (
int(config.get('timeout'))
if 'timeout' in config.keys() else None
)
tries = int(config.get('tries', 1))
scraper = BCRASMLScraper(
url=config.get('url'),
timeout=timeout,
tries=tries,
coins=config.get('coins'),
types=config.get('types'),
skip_intermediate_panel_data=skip_intermediate_panel_data,
intermediate_panel_path=intermediate_panel_path,
skip_clean_last_dates=skip_clean_last_dates
)
parsed = scraper.run(start_date, end_date)
if parsed:
for k in parsed.keys():
if k == 'peso_uruguayo':
csv_header = ['indice_tiempo']
csv_header.extend(config['types']['peso_uruguayo'].values())
write_file(csv_header, parsed['peso_uruguayo'], peso_uruguayo_file_path)
elif k == 'real':
csv_header = ['indice_tiempo']
csv_header.extend(config['types']['real'].values())
write_file(csv_header, parsed['real'], real_file_path)
else:
click.echo("No se encontraron resultados")
execution_end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Email().send_validation_group_email(execution_start_time, execution_end_time, start_date, end_date, skip_intermediate_panel_data, identifier='sml')
except InvalidConfigurationError as err:
click.echo(err)
