def __init__(self, nthreads, db,
request_function,
base_request_settings):
# blocking queue
self.request_queue = mp.Queue()
self.item_queue = mp.Queue()
self.worker_pool = []
# synchronized counter and total requests
self.sync_counter = mp.Value('i', 0)
self.sync_tot = mp.Value('i', 0)
self.db_worker = DBWriterWorker(db, self.item_queue, self.sync_counter, self.sync_tot)
self.init_workers(nthreads, request_function, base_request_settings)
self.timer = None
self.last_request = None
class Session:
def __init__(self, nthreads, db,
request_function,
base_request_settings):
# blocking queue
self.request_queue = mp.Queue()
self.item_queue = mp.Queue()
self.worker_pool = []
# synchronized counter and total requests
self.sync_counter = mp.Value('i', 0)
self.sync_tot = mp.Value('i', 0)
self.db_worker = DBWriterWorker(db, self.item_queue, self.sync_counter, self.sync_tot)
self.init_workers(nthreads, request_function, base_request_settings)
self.timer = None
self.last_request = None
def init_workers(self, nthreads, request_func, request_settings):
try:
# # Request with start date:
# request_settings = {
# # 'ProductId': product_ids,
# 'keywords': u'Nvidia gtx',
# 'itemFilter': [
# {'name': 'Condition',
# 'value': 'Used'},
# {'name': 'ListingType',
# 'value': 'Auction'},
# {'name': 'StartTimeFrom',
# 'value': '2015-11-01T00:00:00.000Z',
# },
# ],
# }
# initialize workers
self.db_worker.start()
for i in range(0, nthreads):
worker = ApiWorker(i,
Connection(config_file='ebayconf.yaml'),
request_func,
request_settings,
self.request_queue,
self.item_queue,
self.sync_counter,
self.sync_tot)
self.worker_pool.append(worker)
worker.start()
except ConnectionError as e:
print(e)
print(e.response.dict)
else:
print('Initialized ' + str(nthreads) + ' workers.')
def process_requests(self, requests):
print('Processing requests...')
self.sync_tot.value = len(requests)
self.sync_counter.value = 0
for request in requests:
self.request_queue.put(request)
def __process_requests_with_time(self, requests):
if self.last_request is not None:
requests = list(requests)
for request in requests:
request['dict']['itemFilter'] = [{'name': 'StartTimeFrom',
'value': self.last_request,
}]
self.last_request = time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime(time.time()))
self.process_requests(requests)
def schedule_timer(self, interval, requests):
print('Scheduled data accumulation. dT: ' + str(timedelta(seconds=interval)))
self.timer = PerpetualTimer(interval, self.__process_requests_with_time, [requests])
self.timer.start()
def join(self):
try:
self.db_worker.join()
for worker in self.worker_pool:
worker.join()
except (KeyboardInterrupt, SystemExit):
print('Waiting for threads to finish...')
if self.timer:
self.timer.cancel()
raise
