def test_url_errors(self):
nyt = NYT(self.API_KEY)
nyt.on_no_connection = MagicMock()
c = nyt.search('slovenia')
self.assertIsNone(c)
self.assertEqual(nyt.on_no_connection.call_count, 1)
nyt.on_no_connection = None
with self.assertRaises(URLError):
nyt.search('slovenia')
def test_url_errors(self):
nyt = NYT(self.API_KEY)
nyt.on_no_connection = MagicMock()
c = nyt.search('slovenia')
self.assertIsNone(c)
self.assertEqual(nyt.on_no_connection.call_count, 1)
nyt.on_no_connection = None
with self.assertRaises(URLError):
nyt.search('slovenia')
class NYTTestsErrorRaising(unittest.TestCase):
API_KEY = 'api_key'
def setUp(self):
self.nyt = NYT(self.API_KEY)
def test_url_errors(self):
self.nyt.on_no_connection = MagicMock()
c = self.nyt.search('slovenia')
self.assertIsNone(c)
self.assertEqual(self.nyt.on_no_connection.call_count, 1)
self.nyt.on_no_connection = None
with self.assertRaises(URLError):
self.nyt.search('slovenia')
def test_error_callbacks(self):
n_calls_rate_limit = 0
n_calls_error = 0
def on_rate_limit():
nonlocal n_calls_rate_limit
n_calls_rate_limit += 1
def on_error(e):
nonlocal n_calls_error
n_calls_error += 1
nyt = NYT(self.API_KEY)
nyt.on_rate_limit = on_rate_limit
nyt.on_error = on_error
# both callback, should call rate limit
nyt.search('slovenia')
self.assertEqual(n_calls_rate_limit, 1)
self.assertEqual(n_calls_error, 0)
# only error callback, should call it
n_calls_rate_limit = 0
n_calls_error = 0
nyt.on_rate_limit = None
nyt.search('slovenia')
self.assertEqual(n_calls_rate_limit, 0)
self.assertEqual(n_calls_error, 1)
# no callback
n_calls_rate_limit = 0
n_calls_error = 0
nyt.on_error = None
nyt.on_rate_limit = None
nyt.search('slovenia')
self.assertEqual(n_calls_rate_limit, 0)
self.assertEqual(n_calls_error, 0)
def test_error_callbacks(self):
n_calls_rate_limit = 0
n_calls_error = 0
def on_rate_limit():
nonlocal n_calls_rate_limit
n_calls_rate_limit += 1
def on_error(e):
nonlocal n_calls_error
n_calls_error += 1
nyt = NYT(self.API_KEY)
nyt.on_rate_limit = on_rate_limit
nyt.on_error = on_error
# both callback, should call rate limit
nyt.search('slovenia')
self.assertEqual(n_calls_rate_limit, 1)
self.assertEqual(n_calls_error, 0)
# only error callback, should call it
n_calls_rate_limit = 0
n_calls_error = 0
nyt.on_rate_limit = None
nyt.search('slovenia')
self.assertEqual(n_calls_rate_limit, 0)
self.assertEqual(n_calls_error, 1)
# no callback
n_calls_rate_limit = 0
n_calls_error = 0
nyt.on_error = None
nyt.on_rate_limit = None
nyt.search('slovenia')
self.assertEqual(n_calls_rate_limit, 0)
self.assertEqual(n_calls_error, 0)
class NYTTests(unittest.TestCase):
API_KEY = 'api_key'
def setUp(self):
# NamedTemporaryFile actually creates and opens the file. On windows you can not open it a second time.
# More: https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile
self.tmp = tempfile.NamedTemporaryFile(delete=False)
self.tmp.close()
os.unlink(self.tmp.name)
self.nyt = NYT(self.API_KEY)
self.nyt.cache_path = self.tmp.name
def tearDown(self):
cache_path = f'{self.nyt.cache_path}.db'
self.tmp.close()
if os.path.exists(cache_path):
os.unlink(cache_path)
def test_nyt_key(self):
self.assertTrue(self.nyt.api_key_valid())
def test_nyt_query_keywords(self):
c = self.nyt.search('slovenia', max_docs=10)
self.assertIsInstance(c, Corpus)
self.assertEqual(len(c), 10)
def test_nyt_query_date_range(self):
from_date = datetime.date(2013, 1, 1)
to_date = datetime.date(2014, 1, 1)
corpus = self.nyt.search('slovenia', from_date, to_date, max_docs=10)
self.assertEqual(len(corpus), 10)
time_index = next(i for i, (var, _) in enumerate(NYT.metas)
if isinstance(var, TimeVariable))
tv = corpus.domain.metas[time_index]
for doc in corpus:
date = tv.repr_val(doc.metas[time_index])
date = datetime.datetime.strptime(date, '%Y-%m-%d %H:%M:%S').date()
self.assertGreaterEqual(date, from_date)
self.assertLessEqual(date, to_date)
def test_nyt_query_max_records(self):
c = self.nyt.search('slovenia', max_docs=25)
self.assertEqual(len(c), 25)
def test_nyt_corpus_domain_generation(self):
corpus = self.nyt.search('slovenia', max_docs=10)
for var, _ in NYT.attributes:
self.assertIn(var, corpus.domain.attributes)
for var, _ in NYT.class_vars:
self.assertIn(var, corpus.domain.class_vars)
for var, _ in NYT.metas:
self.assertIn(var, corpus.domain.metas)
def test_nyt_result_caching(self):
self.nyt._fetch_page('slovenia', None, None, 0) # assure in cache
_, go_sleep = self.nyt._fetch_page('slovenia', None, None, 0)
self.assertFalse(go_sleep)
def test_nyt_sleep(self):
self.nyt.search('slovenia', max_docs=25)
with patch('time.sleep', Mock()) as sleep:
self.nyt.search('slovenia', max_docs=25)
self.assertEqual(sleep.call_count, 0)
def test_on_progress(self):
n_calls = 0
def on_progress(progress, max_docs):
nonlocal n_calls
n_calls += 1
self.assertEqual(max_docs, 25)
self.nyt.search('slovenia', max_docs=25, on_progress=on_progress)
self.assertEqual(n_calls, 3)
def test_break(self):
def should_break():
return True
c = self.nyt.search('slovenia', max_docs=25, should_break=should_break)
self.assertEqual(len(c), BATCH_SIZE)
def test_error_empty_result(self):
nyt = NYT(self.API_KEY)
c = nyt.search('slovenia', max_docs=25)
self.assertIsNone(c)
class NYTTests(unittest.TestCase):
API_KEY = 'api_key'
def setUp(self):
self.tmp = tempfile.NamedTemporaryFile(delete=False)
os.remove(self.tmp.name)
self.nyt = NYT(self.API_KEY)
self.nyt.cache_path = self.tmp.name
def tearDown(self):
if os.path.exists(self.tmp.name):
os.remove(self.tmp.name)
def test_nyt_key(self):
self.assertTrue(self.nyt.api_key_valid())
def test_nyt_query_keywords(self):
c = self.nyt.search('slovenia', max_docs=10)
self.assertIsInstance(c, Corpus)
self.assertEqual(len(c), 10)
def test_nyt_query_date_range(self):
from_date = datetime.date(2013, 1, 1)
to_date = datetime.date(2014, 1, 1)
corpus = self.nyt.search('slovenia', from_date, to_date, max_docs=10)
self.assertEqual(len(corpus), 10)
time_index = next(i for i, (var, _) in enumerate(NYT.metas) if isinstance(var, TimeVariable))
tv = corpus.domain.metas[time_index]
for doc in corpus:
date = tv.repr_val(doc.metas[time_index])
date = datetime.datetime.strptime(date, '%Y-%m-%d %H:%M:%S').date()
self.assertGreaterEqual(date, from_date)
self.assertLessEqual(date, to_date)
def test_nyt_query_max_records(self):
c = self.nyt.search('slovenia', max_docs=25)
self.assertEqual(len(c), 25)
def test_nyt_corpus_domain_generation(self):
corpus = self.nyt.search('slovenia', max_docs=10)
for var, _ in NYT.attributes:
self.assertIn(var, corpus.domain.attributes)
for var, _ in NYT.class_vars:
self.assertIn(var, corpus.domain.class_vars)
for var, _ in NYT.metas:
self.assertIn(var, corpus.domain.metas)
def test_nyt_result_caching(self):
self.nyt._fetch_page('slovenia', None, None, 0) # assure in cache
_, is_cached = self.nyt._fetch_page('slovenia', None, None, 0)
self.assertTrue(is_cached)
def test_on_progress(self):
n_calls = 0
def on_progress(progress, max_docs):
nonlocal n_calls
n_calls += 1
self.assertEqual(max_docs, 25)
self.nyt.search('slovenia', max_docs=25, on_progress=on_progress)
self.assertEqual(n_calls, 3)
def test_break(self):
def should_break():
return True
c = self.nyt.search('slovenia', max_docs=25, should_break=should_break)
self.assertEqual(len(c), BATCH_SIZE)
def test_error_empty_result(self):
nyt = NYT(self.API_KEY)
c = nyt.search('slovenia', max_docs=25)
self.assertIsNone(c)
