def batchsize(page): """validate batchsize""" try: int(page) except: raise ValueError(page + " is not an integer") if int(page) > settings.pagination_limit(): raise ValueError("you have exceeded the batchsize limitation")
def batch(self, date): """get all the events""" batched = [] paginations = list(range(1, settings.pagination_limit() + 1)) per_page = str(settings.per_page()) try: for page in paginations: data = self.get(per_page, date, page) if not data["events"]: return batched batched.extend(data["events"]) return batched except KeyboardInterrupt: print "Caught KeyboardInterrupt, exiting" sys.exit()
def batch(self, date): """multiprocessing to get all the events""" batched = [] mpool = multiprocessing.Pool(settings.threads(), self.utility.init_worker) paginations = list(range(1, settings.pagination_limit() + 1)) per_page = str(settings.per_page()) try: partial_get = partial(self.get, per_page, date) data = mpool.map(partial_get, paginations) for i in data: batched.extend(i["events"]) return batched except KeyboardInterrupt: print "Caught KeyboardInterrupt, terminating workers" mpool.terminate() mpool.join()
def parse_pagination_limit(self): """determine pagination_limit""" if self.options['batchsize'] is None: return settings.pagination_limit() validate.batchsize(self.options['batchsize']) return int(self.options['batchsize'])