def run(self):
i = 0
while True:
try:
command, parameter = self.task_queue.get(timeout=2 * Common.GLOBAL_TIMEOUT)
if command == "die":
break
if i % 100 == 0:
self.logger.info("{} parameter. Left: {}".format(i, self.task_queue.qsize()))
row = self.connection_to_4sq.get_venue(parameter)['venue']
Common.addCategory(row, self.categories)
self.writer_queue.put(("write", row), timeout=2 * Common.GLOBAL_TIMEOUT)
i += 1
except KeyboardInterrupt:
break
except Exception as e:
self.logger.error('{} \n {}'.format(e.args, traceback.format_exc()))
self.logger.warn('die')
self.writer_queue.put(("die", None))
def search_in_area(self, search_parameter):
rows = self.connection_to_4sq.search(search_parameter)['venues']
length = len(rows)
save = True
if length == 50:
save = False
if ((search_parameter.northPoint - search_parameter.southPoint) < 0.001 and
(search_parameter.westPoint - search_parameter.eastPoint) < 0.001):
self.logger.warn('Deepest params: {}'.format(search_parameter))
save = True
else:
self.logger.debug('Too many objects in area {} {} of 50 limit'.format(search_parameter, length))
new_parameters = search_parameter.split(True)
new_parameters = [x for param in new_parameters for x in param.split(False)]
for param in new_parameters:
self.search_in_area(param)
if save:
for row in rows:
Common.addCategory(row, self.categories)
self.writer_queue.put(("write_ids", row))
