def scraper(): http_client = None try: http_client = cache_requests.Session(connection=redis.fromurl('redis://localhost:6379/punters_client_test')) except BaseException: try: http_client = cache_requests.Session() except BaseException: http_client = requests.Session() html_parser = html.fromstring return punters_client.Scraper(http_client, html_parser)
def __init__(self, backup_database=False, cache_expiry=600, database_name='predictivepunter', logging_level=logging.INFO, message_prefix='processing', threads=4, *args, **kwargs): """Initialize instance dependencies""" self.backup_database = backup_database self.cache_expiry = cache_expiry self.database_name = database_name self.logging_level = logging_level logging.basicConfig(level=self.logging_level) self.database = pymongo.MongoClient()[self.database_name] self.database_has_changed = False self.http_client = cache_requests.Session(ex=self.cache_expiry) self.html_parser = html.fromstring self.scraper = pypunters.Scraper(self.http_client, self.html_parser) pyracing.initialize(self.database, self.scraper) for entity in (Seed, Prediction): entity.initialize() for entity in ('meet', 'race', 'runner', 'horse', 'jockey', 'trainer', 'performance', 'seed', 'prediction'): pyracing.add_subscriber('saved_' + entity, self.handle_saved_event) super().__init__(threads=threads, message_prefix=message_prefix)
def provider(database): http_client = None try: http_client = cache_requests.Session(connection=redis.fromurl('redis://localhost:6379/predictive_punter_test')) except BaseException: try: http_client = cache_requests.Session() except BaseException: http_client = requests.Session() html_parser = html.fromstring scraper = punters_client.Scraper(http_client, html_parser) return predictive_punter.Provider(database, scraper)
def provider(): database_uri = 'mongodb://localhost:27017/racing_data_test' database_name = database_uri.split('/')[-1] database_client = pymongo.MongoClient(database_uri) database_client.drop_database(database_name) database = database_client.get_default_database() http_client = None try: http_client = cache_requests.Session(connection=redis.fromurl( 'redis://localhost:6379/racing_data_test')) except BaseException: try: http_client = cache_requests.Session() except BaseException: http_client = requests.Session() html_parser = html.fromstring scraper = punters_client.Scraper(http_client, html_parser) return racing_data.Provider(database, scraper)
def __init__(self, *args, **kwargs): logging.basicConfig(level=kwargs['logging_level']) database_client = pymongo.MongoClient(kwargs['database_uri']) self.database = database_client.get_default_database() self.do_database_backups = kwargs['backup_database'] http_client = None try: http_client = cache_requests.Session( connection=redis.fromurl(kwargs['redis_uri'])) except BaseException: try: http_client = cache_requests.Session() except BaseException: http_client = requests.Session() html_parser = html.fromstring scraper = punters_client.Scraper(http_client, html_parser) self.provider = Provider(self.database, scraper)
def setUpClass(cls): cls.http_client = cache_requests.Session() cls.html_parser = html.fromstring cls.scraper = pypunters.Scraper(cls.http_client, cls.html_parser)
from datetime import datetime, timedelta import unittest import cache_requests from lxml import html import pymongo import pypunters import pyracing database = pymongo.MongoClient()['pyracing_test'] http_client = cache_requests.Session() html_parser = html.fromstring scraper = pypunters.Scraper(http_client, html_parser) pyracing.initialize(database, scraper) historical_date = datetime(2016, 2, 1) future_date = datetime.today().replace( hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1) class EntityTest(unittest.TestCase): def check_ids(self, collection): """Check that all items in the collection have a database ID""" for item in collection: self.assertIn('_id', item) self.assertIsNotNone(item['_id']) def check_no_rescrape(self, get_method, *get_args, **get_kwargs):