Пример #1
0
def index():
    item_reader = ItemReader()
    indexer = ItemIndexer(_api_client, _index_name)
    
    curr_millis = _start_millis
    while True:
        try:
            snapshot_resp = fetch('https://api.mercadolibre.com/items/snapshot/%s' % curr_millis)
            json_str = snapshot_resp.content
            json_obj = anyjson.deserialize(json_str)
            
            for item_id in json_obj:
                try:
                    item = item_reader.get_item(item_id)
                    item.indexed = False
                    item.save()
                    indexer.index(item)
                    indexer.update_categories(item)
                    indexer.update_variables(item)
                    item.indexed = True
                    item.save()
                    print 'item indexed %s' % item_id
                
                except Exception, e:
                    print e

            curr_millis += 1000
        except Exception, e:
            print e
Пример #2
0
class ItemReaderThread:

    def __init__(self, ids_queue, items_queue, http_pool):
        self.__ids_queue = ids_queue
        self.__items_queue = items_queue
        self.__http_pool = http_pool
        self.__item_reader = ItemReader(self.__http_pool)

    def retrieve(self):
        global items_retrieved
        while True:
            items_retrieved += 1
            item_id = self.__ids_queue.get()
            item = self.__item_reader.get_item(item_id)
            self.__items_queue.put(item)
            self.__ids_queue.task_done()
Пример #3
0
 def __init__(self, ids_queue, items_queue, http_pool):
     self.__ids_queue = ids_queue
     self.__items_queue = items_queue
     self.__http_pool = http_pool
     self.__item_reader = ItemReader(self.__http_pool)