def batchsize(page):
"""validate batchsize"""
try:
int(page)
except:
raise ValueError(page + " is not an integer")
if int(page) > settings.pagination_limit():
raise ValueError("you have exceeded the batchsize limitation")
def batchsize(page):
"""validate batchsize"""
try:
int(page)
except:
raise ValueError(page + " is not an integer")
if int(page) > settings.pagination_limit():
raise ValueError("you have exceeded the batchsize limitation")
def batch(self, date):
"""get all the events"""
batched = []
paginations = list(range(1, settings.pagination_limit() + 1))
per_page = str(settings.per_page())
try:
for page in paginations:
data = self.get(per_page, date, page)
if not data["events"]:
return batched
batched.extend(data["events"])
return batched
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, exiting"
sys.exit()
def batch(self, date):
"""multiprocessing to get all the events"""
batched = []
mpool = multiprocessing.Pool(settings.threads(), self.utility.init_worker)
paginations = list(range(1, settings.pagination_limit() + 1))
per_page = str(settings.per_page())
try:
partial_get = partial(self.get, per_page, date)
data = mpool.map(partial_get, paginations)
for i in data:
batched.extend(i["events"])
return batched
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, terminating workers"
mpool.terminate()
mpool.join()
def parse_pagination_limit(self):
"""determine pagination_limit"""
if self.options['batchsize'] is None:
return settings.pagination_limit()
validate.batchsize(self.options['batchsize'])
return int(self.options['batchsize'])
def parse_pagination_limit(self):
"""determine pagination_limit"""
if self.options['batchsize'] is None:
return settings.pagination_limit()
validate.batchsize(self.options['batchsize'])
return int(self.options['batchsize'])