def scraper():
http_client = None
try:
http_client = cache_requests.Session(connection=redis.fromurl('redis://localhost:6379/punters_client_test'))
except BaseException:
try:
http_client = cache_requests.Session()
except BaseException:
http_client = requests.Session()
html_parser = html.fromstring
return punters_client.Scraper(http_client, html_parser)
def __init__(self,
backup_database=False,
cache_expiry=600,
database_name='predictivepunter',
logging_level=logging.INFO,
message_prefix='processing',
threads=4,
*args,
**kwargs):
"""Initialize instance dependencies"""
self.backup_database = backup_database
self.cache_expiry = cache_expiry
self.database_name = database_name
self.logging_level = logging_level
logging.basicConfig(level=self.logging_level)
self.database = pymongo.MongoClient()[self.database_name]
self.database_has_changed = False
self.http_client = cache_requests.Session(ex=self.cache_expiry)
self.html_parser = html.fromstring
self.scraper = pypunters.Scraper(self.http_client, self.html_parser)
pyracing.initialize(self.database, self.scraper)
for entity in (Seed, Prediction):
entity.initialize()
for entity in ('meet', 'race', 'runner', 'horse', 'jockey', 'trainer',
'performance', 'seed', 'prediction'):
pyracing.add_subscriber('saved_' + entity, self.handle_saved_event)
super().__init__(threads=threads, message_prefix=message_prefix)
def provider(database):
http_client = None
try:
http_client = cache_requests.Session(connection=redis.fromurl('redis://localhost:6379/predictive_punter_test'))
except BaseException:
try:
http_client = cache_requests.Session()
except BaseException:
http_client = requests.Session()
html_parser = html.fromstring
scraper = punters_client.Scraper(http_client, html_parser)
return predictive_punter.Provider(database, scraper)
def provider():
database_uri = 'mongodb://localhost:27017/racing_data_test'
database_name = database_uri.split('/')[-1]
database_client = pymongo.MongoClient(database_uri)
database_client.drop_database(database_name)
database = database_client.get_default_database()
http_client = None
try:
http_client = cache_requests.Session(connection=redis.fromurl(
'redis://localhost:6379/racing_data_test'))
except BaseException:
try:
http_client = cache_requests.Session()
except BaseException:
http_client = requests.Session()
html_parser = html.fromstring
scraper = punters_client.Scraper(http_client, html_parser)
return racing_data.Provider(database, scraper)
def __init__(self, *args, **kwargs):
logging.basicConfig(level=kwargs['logging_level'])
database_client = pymongo.MongoClient(kwargs['database_uri'])
self.database = database_client.get_default_database()
self.do_database_backups = kwargs['backup_database']
http_client = None
try:
http_client = cache_requests.Session(
connection=redis.fromurl(kwargs['redis_uri']))
except BaseException:
try:
http_client = cache_requests.Session()
except BaseException:
http_client = requests.Session()
html_parser = html.fromstring
scraper = punters_client.Scraper(http_client, html_parser)
self.provider = Provider(self.database, scraper)
def setUpClass(cls):
cls.http_client = cache_requests.Session()
cls.html_parser = html.fromstring
cls.scraper = pypunters.Scraper(cls.http_client, cls.html_parser)
from datetime import datetime, timedelta
import unittest
import cache_requests
from lxml import html
import pymongo
import pypunters
import pyracing
database = pymongo.MongoClient()['pyracing_test']
http_client = cache_requests.Session()
html_parser = html.fromstring
scraper = pypunters.Scraper(http_client, html_parser)
pyracing.initialize(database, scraper)
historical_date = datetime(2016, 2, 1)
future_date = datetime.today().replace(
hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1)
class EntityTest(unittest.TestCase):
def check_ids(self, collection):
"""Check that all items in the collection have a database ID"""
for item in collection:
self.assertIn('_id', item)
self.assertIsNotNone(item['_id'])
def check_no_rescrape(self, get_method, *get_args, **get_kwargs):