Beispiel #1
0
def batchsize(page):
    """validate batchsize"""
    try:
        int(page)
    except:
        raise ValueError(page + " is not an integer")
    if int(page) > settings.pagination_limit():
        raise ValueError("you have exceeded the batchsize limitation")
Beispiel #2
0
def batchsize(page):
    """validate batchsize"""
    try:
        int(page)
    except:
        raise ValueError(page + " is not an integer")
    if int(page) > settings.pagination_limit():
        raise ValueError("you have exceeded the batchsize limitation")
Beispiel #3
0
    def batch(self, date):
        """get all the events"""
        batched = []
        paginations = list(range(1, settings.pagination_limit() + 1))
        per_page = str(settings.per_page())

        try:
            for page in paginations:
                data = self.get(per_page, date, page)
                if not data["events"]:
                    return batched
                batched.extend(data["events"])
            return batched
        except KeyboardInterrupt:
            print "Caught KeyboardInterrupt, exiting"
            sys.exit()
Beispiel #4
0
    def batch(self, date):
        """multiprocessing to get all the events"""

        batched = []
        mpool = multiprocessing.Pool(settings.threads(), self.utility.init_worker)
        paginations = list(range(1, settings.pagination_limit() + 1))
        per_page = str(settings.per_page())

        try:
            partial_get = partial(self.get, per_page, date)
            data = mpool.map(partial_get, paginations)
            for i in data:
                batched.extend(i["events"])
            return batched
        except KeyboardInterrupt:
            print "Caught KeyboardInterrupt, terminating workers"
            mpool.terminate()
            mpool.join()
Beispiel #5
0
 def parse_pagination_limit(self):
     """determine pagination_limit"""
     if self.options['batchsize'] is None:
         return settings.pagination_limit()
     validate.batchsize(self.options['batchsize'])
     return int(self.options['batchsize'])
Beispiel #6
0
 def parse_pagination_limit(self):
     """determine pagination_limit"""
     if self.options['batchsize'] is None:
         return settings.pagination_limit()
     validate.batchsize(self.options['batchsize'])
     return int(self.options['batchsize'])