def index():
item_reader = ItemReader()
indexer = ItemIndexer(_api_client, _index_name)
curr_millis = _start_millis
while True:
try:
snapshot_resp = fetch('https://api.mercadolibre.com/items/snapshot/%s' % curr_millis)
json_str = snapshot_resp.content
json_obj = anyjson.deserialize(json_str)
for item_id in json_obj:
try:
item = item_reader.get_item(item_id)
item.indexed = False
item.save()
indexer.index(item)
indexer.update_categories(item)
indexer.update_variables(item)
item.indexed = True
item.save()
print 'item indexed %s' % item_id
except Exception, e:
print e
curr_millis += 1000
except Exception, e:
print e
class ItemReaderThread:
def __init__(self, ids_queue, items_queue, http_pool):
self.__ids_queue = ids_queue
self.__items_queue = items_queue
self.__http_pool = http_pool
self.__item_reader = ItemReader(self.__http_pool)
def retrieve(self):
global items_retrieved
while True:
items_retrieved += 1
item_id = self.__ids_queue.get()
item = self.__item_reader.get_item(item_id)
self.__items_queue.put(item)
self.__ids_queue.task_done()
def __init__(self, ids_queue, items_queue, http_pool):
self.__ids_queue = ids_queue
self.__items_queue = items_queue
self.__http_pool = http_pool
self.__item_reader = ItemReader(self.__http_pool)
