Example #1
0
class Session:
    def __init__(self, nthreads, db,
                 request_function,
                 base_request_settings):
        # blocking queue
        self.request_queue = mp.Queue()
        self.item_queue = mp.Queue()
        self.worker_pool = []

        # synchronized counter and total requests
        self.sync_counter = mp.Value('i', 0)
        self.sync_tot = mp.Value('i', 0)

        self.db_worker = DBWriterWorker(db, self.item_queue, self.sync_counter, self.sync_tot)

        self.init_workers(nthreads, request_function, base_request_settings)

        self.timer = None
        self.last_request = None

    def init_workers(self, nthreads, request_func, request_settings):
        try:

            # # Request with start date:
            # request_settings = {
            #     # 'ProductId': product_ids,
            #     'keywords': u'Nvidia gtx',
            #     'itemFilter': [
            #         {'name': 'Condition',
            #          'value': 'Used'},
            #         {'name': 'ListingType',
            #          'value': 'Auction'},
            #         {'name': 'StartTimeFrom',
            #          'value': '2015-11-01T00:00:00.000Z',
            #         },
            #     ],
            # }

            # initialize workers
            self.db_worker.start()

            for i in range(0, nthreads):
                worker = ApiWorker(i,
                                   Connection(config_file='ebayconf.yaml'),
                                   request_func,
                                   request_settings,
                                   self.request_queue,
                                   self.item_queue,
                                   self.sync_counter,
                                   self.sync_tot)

                self.worker_pool.append(worker)
                worker.start()

        except ConnectionError as e:
            print(e)
            print(e.response.dict)
        else:
            print('Initialized ' + str(nthreads) + ' workers.')

    def process_requests(self, requests):

        print('Processing requests...')

        self.sync_tot.value = len(requests)
        self.sync_counter.value = 0

        for request in requests:
            self.request_queue.put(request)

    def __process_requests_with_time(self, requests):

        if self.last_request is not None:
            requests = list(requests)

            for request in requests:
                request['dict']['itemFilter'] = [{'name': 'StartTimeFrom',
                                                  'value': self.last_request,
                                                  }]

        self.last_request = time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime(time.time()))
        self.process_requests(requests)

    def schedule_timer(self, interval, requests):
        print('Scheduled data accumulation. dT: ' + str(timedelta(seconds=interval)))
        self.timer = PerpetualTimer(interval, self.__process_requests_with_time, [requests])
        self.timer.start()

    def join(self):
        try:
            self.db_worker.join()
            for worker in self.worker_pool:
                worker.join()
        except (KeyboardInterrupt, SystemExit):
            print('Waiting for threads to finish...')
            if self.timer:
                self.timer.cancel()
            raise
Example #2
0
 def schedule_timer(self, interval, requests):
     print('Scheduled data accumulation. dT: ' + str(timedelta(seconds=interval)))
     self.timer = PerpetualTimer(interval, self.__process_requests_with_time, [requests])
     self.timer.start()