def run(self): i = 0 while True: try: command, parameter = self.task_queue.get(timeout=2 * Common.GLOBAL_TIMEOUT) if command == "die": break if i % 100 == 0: self.logger.info("{} parameter. Left: {}".format(i, self.task_queue.qsize())) row = self.connection_to_4sq.get_venue(parameter)['venue'] Common.addCategory(row, self.categories) self.writer_queue.put(("write", row), timeout=2 * Common.GLOBAL_TIMEOUT) i += 1 except KeyboardInterrupt: break except Exception as e: self.logger.error('{} \n {}'.format(e.args, traceback.format_exc())) self.logger.warn('die') self.writer_queue.put(("die", None))
def search_in_area(self, search_parameter): rows = self.connection_to_4sq.search(search_parameter)['venues'] length = len(rows) save = True if length == 50: save = False if ((search_parameter.northPoint - search_parameter.southPoint) < 0.001 and (search_parameter.westPoint - search_parameter.eastPoint) < 0.001): self.logger.warn('Deepest params: {}'.format(search_parameter)) save = True else: self.logger.debug('Too many objects in area {} {} of 50 limit'.format(search_parameter, length)) new_parameters = search_parameter.split(True) new_parameters = [x for param in new_parameters for x in param.split(False)] for param in new_parameters: self.search_in_area(param) if save: for row in rows: Common.addCategory(row, self.categories) self.writer_queue.put(("write_ids", row))