def test_get_data_to_check():
# TODO: set_up and tear_down test case
set_envs()
conn = get_conn()
cur = conn.cursor()
delete_tables(cur)
conn.commit()
test_db_credentials = form_db_credentials()
create_tables(test_db_credentials, tables)
for url in ['url1', 'url2']:
cur.execute(
'INSERT INTO urls (url) VALUES (%s) ON CONFLICT DO NOTHING;',
(url, ))
cur.execute(
"INSERT INTO regexps (regexp_text, url_id) VALUES ('.', (SELECT id FROM urls WHERE url='url2'))"
)
conn.commit()
assert get_data_to_check(test_db_credentials) == [('url1', None),
('url2', '.')]
delete_tables(cur)
conn.commit()
cur.close()
conn.close()
def test_form_db_credentials():
os.environ['DBNAME'] = 'test_db'
os.environ['DBUSER'] = '******'
os.environ['DBPASSWORD'] = '******'
os.environ['DBHOST'] = 'test_host'
os.environ['DBPORT'] = 'test_port'
test_db_credentials = form_db_credentials()
assert 'test_db' in test_db_credentials
assert 'test_user' in test_db_credentials
assert 'test_pwd' in test_db_credentials
assert 'test_host' in test_db_credentials
assert 'test_port' in test_db_credentials
def test_get_urls_none():
# TODO: set_up and tear_down test case
set_envs()
conn = get_conn()
cur = conn.cursor()
delete_tables(cur)
conn.commit()
test_db_credentials = form_db_credentials()
create_tables(test_db_credentials, tables)
assert get_data_to_check(test_db_credentials) == []
delete_tables(cur)
conn.commit()
cur.close()
conn.close()
def test_get_sleep_time_none():
# TODO: set_up and tear_down test case
set_envs()
conn = get_conn()
cur = conn.cursor()
delete_tables(cur)
conn.commit()
test_db_credentials = form_db_credentials()
create_tables(test_db_credentials, tables)
assert get_sleep_time(test_db_credentials) == default_timeout
delete_tables(cur)
conn.commit()
cur.close()
conn.close()
def test_get_sleep_time_one():
# TODO: set_up and tear_down test case
set_envs()
conn = get_conn()
cur = conn.cursor()
delete_tables(cur)
conn.commit()
test_db_credentials = form_db_credentials()
create_tables(test_db_credentials, tables)
cur.execute('INSERT INTO timeouts (timeout) VALUES (3);')
conn.commit()
assert get_sleep_time(test_db_credentials) == 3
delete_tables(cur)
conn.commit()
cur.close()
conn.close()
def test_create_tables_one_table():
# TODO: set_up and tear_down test case
set_envs()
conn = get_conn()
cur = conn.cursor()
delete_tables(cur)
conn.commit()
for table in ['foo', 'bar']:
cur.execute(test_tables[table])
conn.commit()
test_db_credentials = form_db_credentials()
create_tables(test_db_credentials, test_tables)
assert table_exists(cur, 'foo_bar')
delete_tables(cur)
conn.commit()
cur.close()
conn.close()
async def consume():
"""
main consumer pipeline
:return: None
"""
db_credentials = form_db_credentials()
create_tables(db_credentials, tables)
message_broker_credentials = form_message_broker_credentials()
consumer = AIOKafkaConsumer(
kafka_topic,
bootstrap_servers=message_broker_credentials,
security_protocol="SSL", ssl_context=context
)
await consumer.start()
try:
async for msg in consumer:
message = json.loads(msg.value)
logger.info(f'saving message {message.get("url", "error")}')
await save_data(db_credentials, message)
finally:
await consumer.stop()
async def test_save_data():
set_envs()
set_envs()
conn = get_conn()
cur = conn.cursor()
delete_tables(cur)
conn.commit()
test_db_credentials = form_db_credentials()
create_tables(test_db_credentials, tables)
test_data = {
'error_code': 200,
'latency': 0.004029075000289595,
'regexp': False,
'url': 'http://localhost:56319/foobar',
'checked_at': '1970-01-01_16:20'
}
await save_data(test_db_credentials, test_data)
cur.execute('SELECT id, url FROM urls;')
url_id, url = cur.fetchone()
assert url == test_data['url']
cur.execute(
'SELECT error_code, latency, regexp, checked_at, url_id FROM check_data;'
)
error_code, latency, regexp, checked_at, url_id_check = cur.fetchone()
assert error_code == test_data['error_code']
assert latency == test_data['latency']
assert regexp == test_data['regexp']
assert checked_at == test_data['checked_at']
assert url_id_check == url_id
delete_tables(cur)
conn.commit()
def main():
"""
Main producer pipeline
:return: None
"""
db_credentials = form_db_credentials()
message_broker_credentials = form_message_broker_credentials()
create_tables(db_credentials, tables)
while True:
data_to_check = get_data_to_check(db_credentials)
time_to_sleep = get_sleep_time(db_credentials)
if not data_to_check:
sleep(time_to_sleep)
logger.info(f'sleeping {time_to_sleep} seconds')
continue
tasks = []
for row in data_to_check:
url, regexp = row
website = Website(url, message_broker_credentials, regexp)
tasks.append(website.perform_check())
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*tasks))
logger.info(f'sleeping {time_to_sleep} seconds')
sleep(time_to_sleep)
