def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: if url == None: return self.srcs try: result = client.request(url, referer=self.base_link) except: result = '' items = client.parseDOM(result, "source", ret="src") for item in items: try: url = item host = client.host(url) self.srcs.append({ 'source': host, 'parts': '1', 'quality': 'HD', 'scraper': self.name, 'url': url, 'direct': False }) except: pass logger.debug('SOURCES [%s]' % self.srcs, __name__) return self.srcs except Exception as e: logger.error('[%s] Exception : %s' % (self.__class__, e)) return self.srcs
def sources(self, url): try: logger.debug('SOURCES URL %s' % url, __name__) quality = 'HD' srcs = [] try: result = client.request(url) except: result = '' result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n','').replace('\t','') result = client.parseDOM(result, "div", attrs={"class": "single-post-video"})[0] items = re.compile('(SRC|src|data-config)=[\'|\"](.+?)[\'|\"]').findall(result) for item in items: if item[1].endswith('png'): continue host = client.host(item[1]) url = item[1] parts = [url] srcs.append({'source':host, 'parts': len(parts), 'quality':quality,'scraper':self.name,'url':"##".join(parts), 'direct':False}) logger.debug('SOURCES [%s]' % srcs, __name__) return srcs except: return srcs
def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] if url == None: return srcs url = urlparse.urljoin(self.base_link, url) try: result = client.request(url, referer=self.base_link) except: result = '' result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n','').replace('\t','') items = client.parseDOM(result, "div", attrs={"class":"entry-content"}) for item in items: try : url = re.compile('(SRC|src|data-config)=[\'|\"](.+?)[\'|\"]').findall(item)[0][1] host = client.host(url) srcs.append({'source': host, 'parts' : '1', 'quality': 'HD', 'scraper': self.name, 'url': url, 'direct':False}) except : pass logger.debug('SOURCES [%s]' % srcs, __name__) return srcs except Exception as e: logger.error('[%s] Exception : %s' % (self.__class__, e)) return srcs
def resolve(url, allowDebrid=False): u = url url = False # Custom Resolvers try: host = client.host(u) r = [i['class'] for i in info() if host in i['host']][0] r = __import__(r, globals(), locals(), [], -1) url = r.resolve(u) if url == False: raise Exception() except: pass # URLResolvers try: if not url == False: raise Exception() logger.debug('Trying URL Resolver for %s' % u, __name__) hmf = urlresolver.HostedMediaFile(url=u, include_disabled=True, include_universal=allowDebrid) if hmf.valid_url() == True: url = hmf.resolve() else: url = False except: pass try: headers = url.rsplit('|', 1)[1] except: headers = '' headers = urllib.quote_plus(headers).replace('%3D', '=').replace( '%26', '&') if ' ' in headers else headers headers = dict(urlparse.parse_qsl(headers)) if url.startswith('http') and '.m3u8' in url: result = client.request(url.split('|')[0], headers=headers, output='geturl', timeout='20') if result == None: raise Exception() elif url.startswith('http'): result = client.request(url.split('|')[0], headers=headers, output='chunk', timeout='20') if result == None: logger.debug('Resolved %s but unable to play' % url, __name__) raise Exception() return url
def source(self, item): try: try: if '720p' in item: quality = 'HD' else: quality = 'SD' urls = client.parseDOM(item, "a", ret="href") for j in range(0, len(urls)): videoID = self.getVideoID(urls[j]) result = client.request(self.info_link % videoID) result = result.decode('iso-8859-1').encode('utf-8') item = client.parseDOM( result, name="div", attrs={ "style": "float:none;height:700px;margin-left:200px" })[0] rUrl = re.compile( '(SRC|src|data-config)=[\'|\"](.+?)[\'|\"]').findall( item)[0][1] if not rUrl.startswith('http:') and not rUrl.startswith( 'https:'): rUrl = '%s%s' % ('http:', rUrl) urls[j] = rUrl host = client.host(urls[0]) url = "##".join(urls) self.srcs.append({ 'source': host, 'parts': str(len(urls)), 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) urls = [] except Exception as e: logger.error(e) pass except: return self.srcs
def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] if url == None: return srcs #url = urlparse.urljoin(self.base_link, url) try: result = client.request(url, referer=self.base_link) except: result = '' result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n', '').replace('\t', '') result = client.parseDOM( result, "div", attrs={"class": "entry-content clearfix single-post-content"}) result = client.parseDOM(result, "p", attrs={"style": "text-align: center;"}) items = client.parseDOM(result, "a", ret="href") for item in items: try: url = item if 'digibolly.se' in url: result = client.request(url) url = re.findall('<iframe src="(.+?)"', result, re.IGNORECASE)[0] host = client.host(url) srcs.append({ 'source': host, 'parts': '1', 'quality': 'HD', 'scraper': self.name, 'url': url, 'direct': False }) except: pass logger.debug('SOURCES [%s]' % srcs, __name__) return srcs except Exception as e: logger.error('[%s] Exception : %s' % (self.__class__, e)) return srcs
def source(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] if url == None: return srcs result, response_code, response_headers, headers, cookie = client.request(url, output='extended') result = result.replace('\n','').replace('\t','').replace('\r','') referer = headers.get('Referer') result = client.parseDOM(result, 'div', attrs={"class":"detail ls_item"})[0] link = client.parseDOM(result, 'div', attrs={"class":"loaer_detai"})[0] link = client.parseDOM(link, 'a', ret='href')[0] link = urlparse.urljoin(referer, link) result = client.request(link) result = re.compile('sources:\s\[(.+?)\]').findall(result)[0] result = '[%s]' % result result = json.loads(result) for item in result: url = item.get('file') label = item.get('label') if '1080p' in label: quality = '1080p' elif '720p' in label : quality = 'HD' elif '360p' in label: quality = 'SD' else: quality = 'SCR' host = client.host(url) srcs.append({'source': host, 'parts' : '1', 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False}) logger.debug('SOURCES [%s]' % srcs, __name__) return srcs except : return srcs
def source(self, item): title = item[0] links = item[1] urls = [] if '720p' in title: quality = 'HD' else: quality = 'SD' parts = client.parseDOM(links, "a", ret="href") srcs = [] for part in parts: try: part = client.request(part) part = part.decode('iso-8859-1').encode('utf-8') part = client.parseDOM( part, "td", attrs={ "style": "vertical-align:middle;text-align:center;" })[0] tUrl = re.compile('(SRC|src|data-config)=[\'|\"](.+?)[\'|\"]' ).findall(part)[0][1] host = client.host(tUrl) urls.append(tUrl) except Exception as e: logger.error(e) pass url = "##".join(urls) srcs.append({ 'source': host, 'parts': len(urls), 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) return srcs
def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] if url == None: return srcs url = urlparse.urljoin(self.base_link, url) try: result = client.request(url, referer=self.base_link) except: result = '' result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n', '').replace('\t', '') result = client.parseDOM(result, "div", attrs={"id": "list-dl"}) items = client.parseDOM(result, "a", ret="href") for item in items: try: url = item host = client.host(url) srcs.append({ 'source': host, 'parts': '1', 'quality': 'HD', 'scraper': self.name, 'url': url, 'direct': False }) except: pass logger.debug('SOURCES [%s]' % srcs, __name__) return srcs except Exception as e: logger.error('[%s] Exception : %s' % (self.__class__, e)) return srcs
def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] if url == None: return srcs if 'hd' in url.lower(): quality = 'HD' else: quality = 'SD' html = client.request(url) mlink = SoupStrainer("div", {"class": "entry"}) videoclass = BeautifulSoup(html, parseOnlyThese=mlink) try: links = videoclass.findAll('iframe') for link in links: url = link.get('src') host = client.host(url) srcs.append({ 'source': host, 'parts': '1', 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) except: pass return srcs except: return srcs
def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] if url == None: return srcs result = client.request(url) result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n', '') result = client.parseDOM( result, name="div", attrs={"class": "entry-content rich-content"})[0] result = client.parseDOM(result, name="p") try: quality = host = '' urls = [] result = result[1::] serversList = result[::2] linksList = result[1::2] for i in range(0, len(serversList)): try: links = linksList[i] urls = client.parseDOM(links, name="a", ret="href") for j in range(0, len(urls)): try: item = client.request(urls[j], mobile=True) item = client.parseDOM(item, "td")[0] item = re.compile( '(SRC|src|data-config)=\"(.+?)\"').findall( item)[0][1] urls[j] = item except: pass if len(urls) > 1: url = "##".join(urls) else: url = urls[0] host = client.host(urls[0]) srcs.append({ 'source': host, 'parts': str(len(urls)), 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) except: pass except: pass logger.debug('SOURCES [%s]' % srcs, __name__) return srcs except: return srcs
def scrape_movie(self, title, year, imdb, debrid=False): try: query = '%s %s' % (title, year) query = self.search_link % (urllib.quote_plus(query)) query = urlparse.urljoin(self.base_link, query) result = client.request(query) result = result.decode('iso-8859-1').encode('utf-8') posts = client.parseDOM(result, "item") items = [] for post in posts: try: t = client.parseDOM(post, 'title')[0] if 'trailer' in cleantitle.get(t): raise Exception() try: s = re.findall( '((?:\d+\.\d+|\d+\,\d+|\d+)(?:GB|GiB|MB|MiB|mb|gb))', t)[0] except: s = '0' i = client.parseDOM(post, 'link')[0] items += [{'name': t, 'url': i, 'size': s}] except: pass title = cleantitle.get(title) for item in items: try: name = item.get('name') t = re.sub( '(\.|\(|\[|\s)(\d{4}|S\d*E\d*|S\d*|3D)(\.|\)|\]|\s|)(.+|)', '', name) if cleantitle.get(title) == cleantitle.get(t): y = re.findall( '[\.|\(|\[|\s](\d{4}|S\d*E\d*|S\d*)[\.|\)|\]|\s]', name)[-1].upper() if not y == year: raise Exception() fmt = re.sub( '(.+)(\.|\(|\[|\s)(\d{4}|S\d*E\d*|S\d*)(\.|\)|\]|\s)', '', name.upper()) fmt = re.split('\.|\(|\)|\[|\]|\s|\-', fmt) fmt = [i.lower() for i in fmt] if any( i.endswith(('subs', 'sub', 'dubbed', 'dub')) for i in fmt): raise Exception() if any(i in ['extras'] for i in fmt): raise Exception() if '1080p' in fmt: quality = '1080p' elif '720p' in fmt: quality = 'HD' else: quality = 'SD' if any(i in ['dvdscr', 'r5', 'r6'] for i in fmt): quality = 'SCR' elif any(i in [ 'camrip', 'tsrip', 'hdcam', 'hdts', 'dvdcam', 'dvdts', 'cam', 'telesync', 'ts' ] for i in fmt): quality = 'CAM' info = [] if '3d' in fmt: info.append('3D') try: size = re.findall( '((?:\d+\.\d+|\d+\,\d+|\d+)(?:GB|GiB|MB|MiB|mb|gb))', item.get('size'))[-1] div = 1 if size.endswith(('GB', 'GiB')) else 1024 size = float(re.sub('[^0-9|/.|/,]', '', size)) / div size = '%.2f GB' % size info.append(size) except: pass if any(i in ['hevc', 'h265', 'x265'] for i in fmt): info.append('HEVC') info = ' | '.join(info) movieurl = item.get('url') result = client.request(movieurl) result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n', '').replace('\t', '') result = client.parseDOM(result, 'div', attrs={'class': 'entry'})[0] links = client.parseDOM(result, 'a', attrs={'target': '_blank'}, ret='href') for link in links: if 'http' in link: host = client.host(link) self.srcs.append({ 'source': host, 'parts': '1', 'quality': quality, 'scraper': self.name, 'url': link, 'direct': False, 'info': info }) except: pass logger.debug('SOURCES [%s]' % self.srcs, __name__) return self.srcs except Exception as e: logger.error('[%s] Exception : %s' % (self.__class__, e)) pass return []
def scrape_episode(self, title, show_year, year, season, episode, imdb, tvdb, debrid=False): try: query = '%s %s' % (title, episode) query = self.search_link % (urllib.quote_plus(query)) result = client.request(self.base_link + query) result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n', '').replace('\t', '') items = client.parseDOM(result, 'content:encoded')[0] items = re.compile('class=\"single-heading\">(.+?)<span').findall( items) for i in range(0, len(items)): try: if '720p' in items[i]: quality = 'HD' else: quality = 'SD' urls = client.parseDOM(items[i], "a", ret="href") for j in range(0, len(urls)): result = client.request(urls[j]) item = BeautifulSoup.BeautifulSoup( result, parseOnlyThese=BeautifulSoup.SoupStrainer( "iframe")) if len(item) == 0: item = re.compile('data-config="(.+?)"').findall( result)[0] item = [{"src": item}] for links in item: rUrl = links["src"] if rUrl.startswith('//'): rUrl = 'http:%s' % rUrl urls[j] = rUrl host = client.host(urls[0]) url = "##".join(urls) self.srcs.append({ 'source': host, 'parts': str(len(urls)), 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) urls = [] except: pass return self.srcs except: return self.srcs
def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] links = [self.base_link_1, self.base_link_2, self.base_link_3] for base_link in links: try: url = url.replace(self.base_link, '') result = client.request(self.base_link + '/' + url) if result == None: continue else: break except: pass result = result.decode('iso-8859-1').encode('utf-8') result = result.replace('\n', '') ### DIRTY Implementation import BeautifulSoup soup = BeautifulSoup.BeautifulSoup(result).findAll( 'blockquote', {'class': re.compile(r'\bpostcontent\b')})[0] for e in soup.findAll('br'): e.extract() if soup.has_key('div'): soup = soup.findChild('div', recursive=False) urls = [] quality = '' for child in soup.findChildren(): if (child.getText() == '') or ( (child.name == 'font' or child.name == 'a') and re.search( 'DesiRulez', str(child.getText()), re.IGNORECASE)): continue elif (child.name == 'font') and re.search( 'Links|Online|Link', str(child.getText()), re.IGNORECASE): if len(urls) > 0: for i in range(0, len(urls)): try: result = client.request(urls[i]) item = client.parseDOM( result, name="div", attrs={ "style": "float:right;margin-bottom:10px" })[0] rUrl = re.compile( '(SRC|src|data-config)=[\'|\"](.+?)[\'|\"]' ).findall(item)[0][1] rUrl = client.urlRewrite(rUrl) urls[i] = rUrl except: urls[i] = client.urlRewrite(urls[i]) pass host = client.host(urls[0]) url = "##".join(urls) srcs.append({ 'source': host, 'parts': str(len(urls)), 'quality': quality, 'provider': 'DesiRulez', 'url': url, 'direct': False }) quality = '' urls = [] quality = child.getText() if '720p HD' in quality: quality = 'HD' elif 'Scr' in quality: quality = 'SCR' else: quality = '' elif (child.name == 'a') and not child.getText() == 'registration': urls.append(str(child['href'])) if quality == '': quality = child.getText() if '720p HD' in quality: quality = 'HD' elif 'Scr' in quality: quality = 'SCR' elif 'Dvd' in quality: quality = 'SD' else: quality = '' if len(urls) > 0: for i in range(0, len(urls)): try: result = client.request(urls[i]) item = client.parseDOM( result, name="div", attrs={"style": "float:right;margin-bottom:10px"})[0] rUrl = re.compile( '(SRC|src|data-config)=[\'|\"](.+?)[\'|\"]' ).findall(item)[0][1] rUrl = client.urlRewrite(rUrl) urls[i] = rUrl except: urls[i] = client.urlRewrite(urls[i]) pass host = client.host(urls[0]) url = "##".join(urls) srcs.append({ 'source': host, 'parts': str(len(urls)), 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) logger.debug('SOURCES [%s]' % srcs, __name__) return srcs except: return srcs
def sources(self, url): logger.debug('SOURCES URL %s' % url, __name__) try: srcs = [] if url == None: return srcs if 'hd' in url.lower(): quality = 'HD' else: quality = 'SD' html = client.request(url) mlink = SoupStrainer('div', {'class': 'entry-content'}) videoclass = BeautifulSoup(html, parseOnlyThese=mlink) try: links = videoclass.findAll('iframe') for link in links: url = link.get('src') host = client.host(url) srcs.append({ 'source': host, 'parts': '1', 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) except: pass try: links = videoclass.findAll('h3') for link in links: url = link.find('a')['href'] host = client.host(url) srcs.append({ 'source': host, 'parts': '1', 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) except: pass try: links = videoclass.findAll('embed') for link in links: url = link.get('src') host = client.host(url) srcs.append({ 'source': host, 'parts': '1', 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) except: pass try: links = videoclass.findAll('a', {'type': 'button'}) for link in links: url = re.findall("(http.*?)'", link.get('onclick'))[0] if 'tv?vq=medium#/' in url: url = url.replace('tv?vq=medium#/', '') host = client.host(url) srcs.append({ 'source': host, 'parts': '1', 'quality': quality, 'scraper': self.name, 'url': url, 'direct': False }) except: pass return srcs except: return srcs