def test_recent_articles(self):
# DOES NOT WORK WITH SQLITE
import settings
if settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.sqlite3':
return
s = amcattest.create_test_set()
sc = Scraper.objects.create(module='amcat.models.scraper',
class_name='TestScraperModel',
articleset=s)
for date in ['2010-01-01'] * 3 + ['2010-01-03'] * 5 + ['2009-01-01'
] * 6:
s.add(amcattest.create_test_article(date=date))
from amcat.tools.toolkit import writeDate
normalize = lambda nn: dict((writeDate(k), v) for (
k,
v,
) in nn.items())
self.assertEqual(normalize(sc.n_scraped_articles()), {
'2010-01-03': 5,
'2010-01-01': 3,
'2009-01-01': 6
})
self.assertEqual(
normalize(sc.n_scraped_articles(from_date='2010-01-01')), {
'2010-01-03': 5,
'2010-01-01': 3
})
s.add(amcattest.create_test_article(date='2010-01-01 13:45'))
self.assertEqual(
normalize(sc.n_scraped_articles(from_date='2010-01-01')), {
'2010-01-03': 5,
'2010-01-01': 4
})
def test_recent_articles(self):
"""DOES NOT WORK WITH SQLITE"""
s = amcattest.create_test_set()
sc = Scraper.objects.create(module='amcat.models.scraper',
class_name='TestScraperModel',
articleset=s)
for date in ['2010-01-01'] * 3 + ['2010-01-03'] * 5 + ['2009-01-01'
] * 6:
s.add(amcattest.create_test_article(date=date))
from amcat.tools.toolkit import writeDate
normalize = lambda nn: dict((writeDate(k), v) for (
k,
v,
) in nn.items())
self.assertEqual(normalize(sc.n_scraped_articles()), {
'2010-01-03': 5,
'2010-01-01': 3,
'2009-01-01': 6
})
self.assertEqual(
normalize(sc.n_scraped_articles(from_date='2010-01-01')), {
'2010-01-03': 5,
'2010-01-01': 3
})
s.add(amcattest.create_test_article(date='2010-01-01 13:45'))
self.assertEqual(
normalize(sc.n_scraped_articles(from_date='2010-01-01')), {
'2010-01-03': 5,
'2010-01-01': 4
})
def test_dateoutput(self):
for date, iso, isotime, yw, ym, yq in ((datetime.datetime(
1990, 1, 10, 13, 1, 0), "1990-01-10", "1990-01-10 13:01:00",
1990.02, 1990.01, 1990.1), ):
self.assertEqual(toolkit.writeDate(date), iso)
self.assertEqual(toolkit.writeDateTime(date), isotime)
self.assertEqual(toolkit.getYW(date), yw)
self.assertEqual(toolkit.getYM(date), ym)
self.assertEqual(toolkit.getYQ(date), yq)
def send_mail(self, result):
table = self.make_table(result).output(rownames = True)
n = sum([r['count'] for r in result])
succesful = sum([r['success'] for r in result])
total = len(result)
datestr = toolkit.writeDate(self.options['date'])
subject = "Daily scraping for {datestr}: {n} articles, {succesful} out of {total} scrapers succesful".format(**locals())
_date = self.options['date']
content = MAIL_ASCII.format(**locals())
for addr in self.options['mail_to'].split(","):
sendmail.sendmail("*****@*****.**",
addr, subject, None, content)
def test_recent_articles(self):
# DOES NOT WORK WITH SQLITE
import settings
if settings.DATABASES["default"]["ENGINE"] == "django.db.backends.sqlite3":
return
s = amcattest.create_test_set()
sc = Scraper.objects.create(module="amcat.models.scraper", class_name="TestScraperModel", articleset=s)
for date in ["2010-01-01"] * 3 + ["2010-01-03"] * 5 + ["2009-01-01"] * 6:
s.add(amcattest.create_test_article(date=date))
from amcat.tools.toolkit import writeDate
normalize = lambda nn: dict((writeDate(k), v) for (k, v) in nn.items())
self.assertEqual(normalize(sc.n_scraped_articles()), {"2010-01-03": 5, "2010-01-01": 3, "2009-01-01": 6})
self.assertEqual(normalize(sc.n_scraped_articles(from_date="2010-01-01")), {"2010-01-03": 5, "2010-01-01": 3})
s.add(amcattest.create_test_article(date="2010-01-01 13:45"))
self.assertEqual(normalize(sc.n_scraped_articles(from_date="2010-01-01")), {"2010-01-03": 5, "2010-01-01": 4})
def send_email(count, messages):
counts = [(s.__class__.__name__, n) for (s,n) in count.items()]
n = sum(count.values())
counts.append(("Total", n))
t = ListTable(counts, colnames=["Scraper", "#Articles"])
counts_ascii = t.output(useunicode=False, box=False)
counts_html = table2html(t, printRowNames=False)
succesful = len([1 for (s,n2) in count.items() if n2>0])
total = len(count.items())
datestr = toolkit.writeDate(date.today())
mail_ascii = MAIL_ASCII.format(table=counts_ascii, **locals())
mail_html = MAIL_HTML.format(table=counts_html, **locals())
subject = "Daily scraping for {datestr}: {n} articles, {succesful} out of {total} scrapers succesful".format(**locals())
sendmail.sendmail("*****@*****.**", EMAIL, subject, mail_html, mail_ascii)
def test_recent_articles(self):
# DOES NOT WORK WITH SQLITE
import settings
if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
return
s = amcattest.create_test_set()
sc =Scraper.objects.create(module='amcat.models.scraper',
class_name='TestScraperModel', articleset=s)
for date in ['2010-01-01'] * 3 + ['2010-01-03'] * 5 + ['2009-01-01'] * 6:
s.add(amcattest.create_test_article(date=date))
from amcat.tools.toolkit import writeDate
normalize = lambda nn : dict((writeDate(k), v) for (k,v,) in nn.items())
self.assertEqual(normalize(sc.n_scraped_articles()),
{'2010-01-03': 5, '2010-01-01': 3, '2009-01-01': 6})
self.assertEqual(normalize(sc.n_scraped_articles(from_date='2010-01-01')),
{'2010-01-03': 5, '2010-01-01': 3})
s.add(amcattest.create_test_article(date='2010-01-01 13:45'))
self.assertEqual(normalize(sc.n_scraped_articles(from_date='2010-01-01')),
{'2010-01-03': 5, '2010-01-01': 4})
def test_dateoutput(self):
for date, iso, isotime, yw, ym, yq in (
(datetime.datetime(1990, 1, 10, 13,1,0), "1990-01-10", "1990-01-10 13:01:00", 1990.02, 1990.01, 1990.1),
):
self.assertEqual(toolkit.writeDate(date), iso)
self.assertEqual(toolkit.writeDateTime(date), isotime)