def _get_images(self, images): def download(remote, local): util.save_data_to_file(util.request(remote), local) not_cached = [(img['remote'], img['local']) for img in images if not os.path.exists(img['local'])] util.run_parallel_in_threads(download, not_cached)
def _fill_categories_parallel(self, list, categories): def process_category(title, url): page = util.request(url) img_m = re.search(CATEGORY_IMG_RE, page, re.IGNORECASE) img = img_m and self._url(img_m.group(1)) page = util.substr(page, LISTING_START, LISTING_END) finditer = False for m in re.finditer(LISTING_ITER_RE, page, re.DOTALL | re.IGNORECASE): finditer = True # payed content if m.group('playimg').find(YELLOW_IMG) != -1: return break # no links if not finditer: return item = self.dir_item() item['title'] = title item['url'] = url item['img'] = img with lock: self._filter(list, item) lock = Lock() util.run_parallel_in_threads(process_category, categories)
def fill_list_parallel(list, matches): def process_match(m): image,plot = self._get_meta(m.group('name'),self._url(m.group('url'))) item = self.dir_item() item['title'] = m.group('name') item['url'] = m.group('url')+'video/' item['img'] = image item['plot'] = plot with lock: self._filter(list, item) lock = Lock() util.run_parallel_in_threads(process_match, matches)
def fill_list_parallel(list, matches): def process_match(m): image, plot = self._get_meta(m.group('name'), self._url(m.group('url'))) item = self.dir_item() item['title'] = m.group('name') item['url'] = m.group('url') + 'video/' item['img'] = image item['plot'] = plot with lock: self._filter(list, item) lock = Lock() util.run_parallel_in_threads(process_match, matches)
def paralel_search(search): def do_search(p,what): res = [] try: result = p.provider.search(what) for item in result: item['title'] = '[%s] %s' % (p.provider.name,item['title']) if item['type'] == 'next': item['type'] = 'dir' item['title'] = '[%s] %s >>' % (p.provider.name,__language__(30063)) except: traceback.print_exc() with lock: p.list(result) lock = Lock() util.run_parallel_in_threads(do_search, search)
def paralel_search(search): def do_search(p,what): res = [] try: result = p.provider.search(what) for item in result: item['title'] = '[%s] %s' % (p.provider.name,item['title']) if item['type'] == 'next': item['type'] = 'dir' item['title'] = '[%s] %s >>' % (p.provider.name,__language__(30063)) except: traceback.print_exc() try: lock.acquire() p.list(result) finally: lock.release() lock = Lock() util.run_parallel_in_threads(do_search, search)
def _request_parallel(self, requests): def fetch(req, *args): return util.request(req), args pages = [] q = util.run_parallel_in_threads(fetch, requests) while True: try: page, args = q.get_nowait() except: break pages.append([page, args]) return pages