def parse_video(self, soup: BeautifulSoup, url: URL): video_container = soup.find('div', id='video') if video_container: video = video_container.find('video', src=True) if video: src = str(video.attrs['src']) self.add_video('default', URL(src)) return frame = video_container.find('iframe', src=True) if frame: src = str(frame.attrs['src']) code = src.rpartition('embed.php?f=')[2] data = {'data': code} loader_url = URL( 'http://donfreeporn.com/wp-content/themes/detube-noedit/Htplugins/Loader.php', method='POST', post_data=data) filedata = FLData(loader_url, '') self._result_type = 'video' self.model.loader.start_load_file(filedata, self.continue_parse_video)
def parse_others(self, soup: BeautifulSoup, url: URL): for thumbnail in _iter( soup.find_all('div', {'class': 'category-thumb'})): # psp(thumbnail.prettify()) try: href = URL(thumbnail.a.attrs['href'], base_url=url) img = thumbnail.img thumb_file = img.attrs.get('data-original', img.attrs.get('src')) thumb_url = URL(thumb_file, base_url=url) label = img.attrs.get('alt', '') duration = thumbnail.find('span') dur_time = '' if duration is None else str(duration.string) hd_div = thumbnail.find('div', {'class': 'hdIcon'}) hd = '' if hd_div is None else ' HD' self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': hd, 'align': 'top left' }, { 'text': label, 'align': 'bottom center' }]) except AttributeError: pass
def parse_thumbs(self, soup: BeautifulSoup, url: URL): container = soup.find('main', {'id': 'main'}) if container: for thumbnail in _iter( container.find_all('div', {'class': 'column'})): # psp(thumbnail.prettify()) xref = thumbnail.find('a') if xref: href = URL(xref.attrs['href'], base_url=url) description = xref.attrs['title'] thumb_url = URL(thumbnail.img.attrs['src'], base_url=url) duration = thumbnail.find('span', {'class': "length"}) dur_time = '' if duration is None else collect_string( duration) quality = thumbnail.find('span', {'class': "quality"}) qual = '' if quality is None else str(quality.string) self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': description, 'align': 'bottom center' }, { 'text': qual, 'align': 'top left', 'bold': True }])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): container = soup.find('div', {'class': 'list-videos'}) if container is not None: for thumbnail in _iter(container.find_all('div', {'class': 'item'})): # psp(thumbnail.prettify()) href = URL(thumbnail.a.attrs['href'], base_url=url) thumb_url = URL(thumbnail.img.attrs['data-original'], base_url=url) label = thumbnail.img.attrs.get('alt', '') duration = thumbnail.find('div', {'class': 'duration'}) dur_time = '' if duration is None else str( duration.contents[-1]) self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for thumbnail in _iter(soup.find_all('div', {'class': 'video'})): href = URL(thumbnail.a.attrs['href'], base_url=url) thumb_url = URL(thumbnail.img.attrs['src'], base_url=url) label = thumbnail.img.attrs.get('alt', '') duration = thumbnail.find('span', {'class': 'video-overlay'}) dur_time = '' if duration is None else str(duration.string) hd_span = thumbnail.find('span', {'class': 'hdmovie-icon'}) hd = 'HD' if hd_span else '' self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }, { 'text': hd, 'align': 'top left' }])
def parse_others(self, soup: BeautifulSoup, url: URL): categories = soup.find('ul', {'class': 'thumbs-categories'}) if categories: for thumbnail in _iter( categories.find_all('li', {'class': 'thumb-category'})): # psp(thumbnail.attrs['class']) # psp(thumbnail.prettify()) href = URL(thumbnail.a.attrs['href'], base_url=url) name = thumbnail.find('span', {'class': 'category-title'}) description = '' if name is None else str(name.string) thumb_url = URL(thumbnail.img.attrs['data-original'], base_url=url) count = thumbnail.find('span', {'class': 'added'}) counter = '' if count is None else str(count.string) self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{ 'text': counter, 'align': 'top right' }, { 'text': description, 'align': 'bottom center' }])
def continue_with_stars(self, filedata: FLData, soup: BeautifulSoup): # parce stars page self.waiting_data = False for stars_container in soup.find_all('ul', {'class': ['pornStarsThumbs']}): for star in _iter(stars_container.find_all('li')): href = URL(star.a.attrs['href'], base_url=filedata.url) img = star.find('img') thumb_url = URL(img.attrs['src'], base_url=filedata.url) label = img.attrs.get('alt', '') num_videos_span = star.find('span', text=lambda x: 'Videos' in str(x)) num_videos = '' if num_videos_span is None else str( num_videos_span.string) self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': num_videos, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }]) self.generate_thumb_view()
def open(self, url: URL) -> bytes: try: headers=dict() headers['user-agent']=url.user_agent if url.referer: headers['Referer']=url.referer.get() if url.method == 'GET': response = requests.get(url.get(), cookies=url.coockies, proxies=self.proxies,headers=headers) elif url.method == 'POST': # print('Loading POST') # print(url.get(), url.post_data) response = requests.post(url.get(), data=url.post_data, proxies=self.proxies,headers=headers) else: raise LoaderError('Unknown method:' + url.method) response.raise_for_status() except requests.exceptions.HTTPError as err: raise LoaderError('HTTP error: {0}'.format(err.response.status_code)) except requests.exceptions.ConnectTimeout: raise LoaderError('Connection timeout') except requests.exceptions.ReadTimeout: raise LoaderError('Read timeout') except requests.exceptions.ConnectionError: raise LoaderError('Connection error') except: raise LoaderError('Unknown error in loader') else: return response.content
def parse_thumbs(self, soup: BeautifulSoup, url: URL): container = soup.find('div', {'class': 'mozaique'}) if container: # psp(container.prettify()) for thumbnail in _iter( container.find_all('div', {'class': 'thumb-block '})): # psp(thumbnail.prettify()) xref = thumbnail.find('a') if xref: # psp(thumbnail.prettify()) href = URL(xref.attrs['href'], base_url=url) script = thumbnail.find('script', text=lambda x: 'img src' in str(x)) thumb_file = quotes(script.text, '<img src="', '"') thumb_url = URL(thumb_file, base_url=url) label = xref.attrs.get('title', '') duration = thumbnail.find('span', {'class': 'duration'}) dur_time = str( duration.string).strip('()') if duration else '' self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for thumbs in _iter(soup.find_all('ul', {'class': 'thumbs'})): for thumbnail in _iter(thumbs.find_all('li')): xref = thumbnail.find('a', {'class': 'thumb'}) if xref: # psp(thumbnail.prettify()) href = URL(xref.attrs['href'], base_url=url) img = thumbnail.img thumb_file = img.attrs.get('data-original', img.attrs.get('src')) thumb_url = URL(thumb_file, base_url=url) label = img.attrs.get('alt', '') duration = thumbnail.find('time') dur_time = '' if duration is None else str( duration.string).strip() self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for post in _iter(soup.find_all('div', {'class':'post_el'})): try: author=post.find('a',{'class':'span_author_name'}) author_str=str(author.span.string).strip() combo=post.find('div',{'class':'combo_post_wrap'}) if combo: href = URL(combo.a.attrs['href'], base_url=url) description = combo.a.attrs['title'] thumb_url = URL(combo.img.attrs['src'], base_url=url) dur_time = 'Multi' else: vid_container = post.find('div', {'class': 'vid_container'}) href = URL(vid_container.a.attrs['href'], base_url=url) description = vid_container.a.attrs['title'] thumb_url = URL(vid_container.img.attrs['src'], base_url=url) duration = post.find('span', {'class': "duration_small"}) dur_time = '' if duration is None else str(duration.string) self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{'text': dur_time, 'align': 'top right'}, {'text': author_str, 'align': 'top left'}, {'text': description, 'align': 'bottom center'}]) except AttributeError: pass
def parse_video(self, soup: BeautifulSoup, url: URL): videos_container = soup.find('div', {'id': 'videos_container'}) video_container = soup.find('div', {'itemprop': 'video'}) if videos_container: usss=soup.find('script',text=lambda x: 'usss' in str(x)) uid=quotes(usss.string.replace(' ',''),'usss[0]="','"') xhr_list=list() part=1 for data_div in _iter(videos_container.find_all('div', {'class':'pl_vid_el'})): data = {'uid': uid, 'source': data_div.attrs['data-source'], 'hash': data_div.attrs['data-hash'], 'x': data_div.attrs['data-x'], 'oid': data_div.attrs['data-oid'], 'pid': data_div.attrs['data-pid']} xhr=URL('/php/get_vlink.php',base_url=url,method='POST',post_data=data) xhr_list.append(YPData(xhr,' - Part {0}'.format(part))) part += 1 loader=self.model.loader load_process=loader.get_new_load_process(self.add_part) self._result_type = 'video' load_process.load_list(xhr_list) elif video_container: video=video_container.find('video') if video: self.add_video('default', URL(video.attrs['src'], base_url=url))
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for thumbnail in _iter(soup.find_all('li', {'class': 'thumb-item'})): href = URL(thumbnail.a.attrs['href'], base_url=url) thumb_url = URL(thumbnail.img.attrs['src'], base_url=url) label = thumbnail.img.attrs.get('alt', '') duration = thumbnail.find('span', {'class': 'fs11 viddata flr'}) dur_time = '' if duration is None else str(duration.contents[-1]) hd_span = thumbnail.find('span', {'class': 'text-active bold'}) hd = '' if hd_span is None else str(hd_span.string) self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }, { 'text': hd, 'align': 'top left' }])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for thumbnail in _iter(soup.find_all('div', {'class': 'video-item'})): private = thumbnail.find('span', {'class': 'ico-private'}) if not private: href = URL(thumbnail.a.attrs['href'], base_url=url) description = thumbnail.img.attrs['alt'] thumb_url = URL(thumbnail.img.attrs['data-original'], base_url=url) duration = thumbnail.find('div', {'class': 'durations'}) dur_time = '' if duration is None else str( duration.contents[-1]).strip() hd_div = thumbnail.find('div', {'class': 'hd-text-icon'}) hd = '' if hd_div is None else str(hd_div.string).strip() self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': description, 'align': 'bottom center' }, { 'text': hd, 'align': 'top left', 'bold': True }])
class HistoryView(QWidget): def __init__(self, parent: QWidget, view_manager: ViewManagerFromViewInterface): super().__init__(parent) self.view_manager = view_manager self.history_items = list() self.current_url = URL() self.history = None self.ui = Ui_HistoryView() savecwd = os.getcwd() os.chdir('view/qt_ui') self.ui.setupUi(self) os.chdir(savecwd) self.ui.combo_history.addItem(self.current_url.get()) self.binding() def binding(self): self.ui.bn_go.clicked.connect(self.on_go_clicked) self.ui.bn_back.clicked.connect(self.on_back_clicked) def update_history(self, history: HistoryFromViewInterface): self.history = history self.ui.combo_history.clear() self.history_items.clear() self.ui.combo_history.addItem(self.current_url.get()) for item in reversed(history.get_history()): self.ui.combo_history.addItem(item.url.get()) self.history_items.append(item) def set_current_url(self, url: URL): self.current_url = url self.ui.combo_history.setItemText(0, self.current_url.get()) def on_go_clicked(self): index = self.ui.combo_history.currentIndex() text = self.ui.combo_history.currentText() if index == 0: if text.strip() == self.current_url.get(): self.view_manager.goto_url(self.current_url) else: self.view_manager.goto_url(URL(text)) else: if text.strip() == self.history_items[index - 1].url.get(): item = self.history_items[index - 1] self.view_manager.goto_url(item.url, {'context': item.context}) else: self.view_manager.goto_url(URL(text)) def on_back_clicked(self): if self.history: item = self.history.back() if item: self.view_manager.goto_url(item.url, { 'context': item.context, 'no_history': True })
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for content in _iter(soup.find_all('div', {'class': 'content'})): for thumbnail in _iter(content.find_all('li', {'class': 'thumb'})): href = URL(thumbnail.a.attrs['href'], base_url=url) name = thumbnail.find('span', {'class': 'thumb-title'}) description = '' if name is None else str(name.string) thumb_url = URL(thumbnail.img.attrs['data-original'], base_url=url) duration = thumbnail.find('span', {'class': ['duration', 'counter']}) dur_time = '' if duration is None else str(duration.string) if dur_time != 'Link': self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': description, 'align': 'bottom center' }])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for thumbnail in _iter( soup.find_all( 'div', {'class': ['video', 'category', 'pornstar', 'serie']})): # psp(thumbnail.prettify()) xref = thumbnail.find('a') if xref: href = URL(xref.attrs['href'], base_url=url) thumb_url = URL(thumbnail.img.attrs['data-original'], base_url=url) label = thumbnail.img.attrs.get('alt', '') duration = thumbnail.find('span', {'class': 'duration'}) dur_time = '' if duration is None else collect_string(duration) hd_span = thumbnail.find('span', {'class': 'hd'}) hd = '' if hd_span is None else str(hd_span.string).strip() self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }, { 'text': hd, 'align': 'top left' }])
def parse_others(self, soup: BeautifulSoup, url: URL): # Categories page categories = dict() for thumbs_container in _iter( soup.find_all('div', {'class': 'list-categories'})): for thumbnail in _iter( thumbs_container.find_all('a', {'class': 'item'})): href = URL(thumbnail.attrs['href'], base_url=url) description = thumbnail.img.attrs['alt'] thumb_url = URL(thumbnail.img.attrs['src'], base_url=url) no = thumbnail.find('div', {'class': 'videos'}) no_of_video = '' if no is None else str(no.string).strip() categories[description] = (href, thumb_url, no_of_video) for description in sorted(categories): (href, thumb_url, no_of_video) = categories[description] self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{ 'text': no_of_video, 'align': 'top right' }, { 'text': description, 'align': 'bottom center' }])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): contents=soup.find('div', {'class':'videos'}) if contents: for thumbnail in _iter(contents.find_all('div', {'class': 'content'})): # psp(thumbnail.prettify()) xref=thumbnail.find('a',href=True) href = URL(xref.attrs['href'], base_url=url) thumb_url = URL(thumbnail.img.attrs['src'], base_url=url) label=thumbnail.img.attrs.get('alt','') duration = thumbnail.find('div', {'class': 'duration'}) dur_time = '' if duration is None else str(duration.string) hd_span = thumbnail.find('span', {'class': 'hd_video'}) hd = 'HD' if hd_span else '' count_em=thumbnail.find('em') count = '' if count_em is None else str(count_em.string) self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{'text':dur_time, 'align':'top right'}, {'text': count, 'align': 'top right'}, {'text':label, 'align':'bottom center'}, {'text': hd, 'align': 'top left'}])
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for thumbnail in _iter( soup.find_all('div', {'class': 'item-inner-col'})): xref = thumbnail.a.attrs['href'] if '/video/' in xref or '/galleries/' in xref: href = URL(thumbnail.a.attrs['href'], base_url=url) description = thumbnail.a.img.attrs['alt'] thumb_file = thumbnail.img.attrs['src'] channel_img = thumbnail.find('img', {'class': "img-responsive"}) thumb_file = thumb_file if channel_img is None else channel_img.attrs[ 'src'] thumb_url = URL(thumb_file, base_url=url) duration = thumbnail.find('span', {'class': "time"}) dur_time = '' if duration is None else str(duration.string) quality = thumbnail.find('span', {'class': "quality-icon"}) qual = '' if quality is None else str(quality.string) self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': qual, 'align': 'top left' }, { 'text': description, 'align': 'bottom center' }])
def parse_video(self, soup: BeautifulSoup, url: URL): player_container = soup.find('div', {'id': 'player'}) if player_container: script = player_container.find( 'script', text=lambda text: 'jwplayer' in str(text)) if script: text = script.string if 'sources:[' in text: sources = quotes(text, 'sources:[', ']') j = json.loads('[' + sources + ']') for j_data in j: if j_data['file'] is not '': self.add_video(j_data['label'], URL(j_data['file'] + '*')) self.set_default_video(-1) elif 'sources:' in text: container = soup.find('div', {'class': 'content-video'}) if container: script = container.find( 'script', {'src': lambda x: '/player/' in str(x)}) if script: script_url = URL(script.attrs['src'], base_url=url) filedata = FLData(script_url, '') self._result_type = 'video' self.model.loader.start_load_file( filedata, self.continue_parse_video)
def parse_thumbs(self, soup: BeautifulSoup, url: URL): for thumb_container in _iter( soup.find_all('div', { 'class': ['box boxTL', 'box boxTR'], 'id': lambda x: x != 'vPromo' })): for thumb in _iter( thumb_container.find_all('div', {'class': 'video'})): # psp(thumb) href = URL(thumb.a.attrs['href'], base_url=url) description = thumb.a.img.attrs['alt'] thumb_url = URL(thumb.img.attrs['src'], base_url=url) duration = thumb.find('b') dur_time = '' if duration is None else str(duration.string) quality = thumb.find('div', {'class': "hSpriteHD"}) qual = '' if quality is None else 'HD' self.add_thumb(thumb_url=thumb_url, href=href, popup=description, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': description, 'align': 'bottom center' }, { 'text': qual, 'align': 'top left', 'bold': True }])
def parse_thumbs_json(self, fldata: FLData): data = loads(fldata.text) if type(data) == dict: page = data['1'] elif type(data) == list: page = data[0] else: page = None try: items = page['items'] for item in items: href = URL(item['video_link'], base_url=fldata.url) label = item['title_long'] dur_time = item['duration'] thumb_url = URL(item['thumb_url'], base_url=fldata.url) self.add_thumb(thumb_url=thumb_url, href=href, popup=label, labels=[{ 'text': dur_time, 'align': 'top right' }, { 'text': label, 'align': 'bottom center' }]) # create pagination navigation = page['navigation'] last_page = navigation['lastPage'] current = navigation['currentPage'] pattern = navigation['urlPattern'].replace('[%pageId%]', '{0}') for page_no in range(current - 5, current + 5): if page_no == current: continue if page_no < 3: continue if page_no > last_page: continue page_url = URL(fldata.url.any_data['first_page_url'].get()) page_url.any_data = dict(json_file_url=URL( pattern.format(page_no), base_url=fldata.url)) self.add_page(str(page_no), page_url) # add last page page_url = URL(fldata.url.any_data['first_page_url'].get()) page_url.any_data = dict(json_file_url=URL( pattern.format(last_page), base_url=fldata.url)) self.add_page(str(last_page), page_url) except TypeError as err: print(dumps(data, sort_keys=True, indent=4)) print('Error:', err) self.waiting_data = False self.generate_thumb_view()
def parse_pictures(self, soup: BeautifulSoup, url: URL): slideshow = soup.find('div', {'id': 'slideshow'}) if slideshow: for slide in _iter(slideshow.find_all('a', {'class': 'gal'})): src = URL(slide.img.attrs['src']) filename = src.get_path() + src.get().rpartition( '/')[2].partition('?')[0] self.add_picture(filename, src)
def parse_pictures(self, soup: BeautifulSoup, url: URL): gallery = soup.find('div', {'class': 'ad-gallery'}) if gallery: for href in _iter(gallery.find_all('a', {'data-image': True})): image_url = URL(href.attrs['data-image']) filename = image_url.get_path() + image_url.get().rpartition( '/')[2] self.add_picture(filename, image_url)
def create_start_button(view:ViewManagerFromModelInterface): menu_items=dict(New=URL('http://www.sextube.nl/videos/nieuw*'), Videos_Top_Rated=URL('http://www.sextube.nl/videos/hoog-gewaardeerd*'), Popular=URL('http://www.sextube.nl/videos/populair*'), HD=URL('http://www.sextube.nl/videos/hd*')) view.add_start_button(picture_filename='model/site/resource/sextube_nl.png', menu_items=menu_items, url=URL("http://www.sextube.nl/videos/nieuw*"))
def create_start_button(view:ViewManagerFromModelInterface): menu_items=dict(Categories=URL('http://pervertslut.com/categories/'), Latest=URL('http://pervertslut.com/latest-updates/'), TopRated=URL('http://pervertslut.com/top-rated/'), MostViewed=URL('http://pervertslut.com/most-popular/')) view.add_start_button(picture_filename='model/site/resource/pervertslut.png', menu_items=menu_items, url=URL("http://pervertslut.com/latest-updates/", test_string='p**n'))
def parse_pictures(self, soup: BeautifulSoup, url: URL): gallery = soup.find('div', {'class': 'gallery-block'}) if gallery: for image in _iter(gallery.find_all('img', src=True)): image_url = URL( str(image.attrs['src']).replace('/thumbs/', '/')) filename = image_url.get_path() + image_url.get().rpartition( '/')[2] self.add_picture(filename, image_url)
def create_start_button(view: ViewManagerFromModelInterface): # menu_items = dict(Top_Rated_Video=URL('http://www.vporn.com/rating'), Latest_Video=URL('http://www.vporn.com/newest'), Longest_Video=URL('https://www.vporn.com/longest/'), HD_video=URL('http://www.vporn.com/newest/hd')) view.add_start_button(picture_filename='model/site/resource/vporn.png', menu_items=menu_items, url=URL("http://www.vporn.com/newest/"))
def parse_video_tags(self, soup: BeautifulSoup, url: URL): author=soup.find('div',{'class':'post_author_name'}) if author: self.add_tag(str(author.span.string), URL(author.a.attrs['href'], base_url=url), style={'color': 'blue'}) for tags_container in _iter(soup.find_all('div',{'class':'popular_block_header_rl'})): for tag in _iter(tags_container.find_all('a')): self.add_tag(str(tag.b.string).strip(),URL(tag.attrs['href'],base_url=url))