def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'thumbs200'),
('div', 'class', 'thumbs300')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', self.get_href)
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'menu')
])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + x)
startpage_pages_rule.set_attribute_filter_function(
'href', lambda txt: '/st/' in txt)
parser.add_rule(startpage_pages_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('div', 'class',
'gallery-thumbs')])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src'})
picture_rule.set_attribute_modifier_function(
'src', self.process_picture_address)
parser.add_rule(picture_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
for f in picture_rule.get_result():
result.add_full(
FullPictureInfo(abs_href=URL(f['src']),
rel_name=f['src'].rpartition('/')[2]))
return result
if len(startpage_rule.get_result()) > 0:
# print('Startpage rule')
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
#
# for item in startpage_hrefs_rule.get_result(['href', 'data']):
# result.add_control(ControlInfo(item['data'], URL(item['href'])))
return result
def parse_picture_page(self, fname, base_url=URL()):
print('Parsing:', base_url.get(), base_url.domain())
parser = SiteParser()
redirect_rule = ParserRule()
redirect_rule.add_activate_rule_level([('head', '', '')])
redirect_rule.add_process_rule_level('base', {'href'})
parser.add_rule(redirect_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('div', 'class', 'girls')])
picture_rule.add_process_rule_level('a', {'href'})
picture_rule.set_attribute_modifier_function('href', lambda text: base_url.get() + '/' + text)
parser.add_rule(picture_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(redirect_rule.get_result()) > 0:
print('Redirecting', redirect_rule.get_result()[0]['href'])
result.set_type('redirect')
result.set_redirect(URL(redirect_rule.get_result()[0]['href']))
return result
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
i = 1
for f in picture_rule.get_result(['href']):
result.add_full(FullPictureInfo(abs_href=URL(f['href']), rel_name='%03d.jpg' % i))
i += 1
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'thumbs'),
('div', 'class', 'movie_thumbs')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', self.get_href)
parser.add_rule(startpage_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(startpage_rule.get_result()) > 0:
# print('Startpage rule')
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']), href=URL(item['href']), popup=item.get('alt', '')))
else:
print(base_url.get(), ' not parsed by BEmultiThumbSite. Add rule.')
return result
def parse_index(self, request, url):
parser = SiteParser()
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('center', '', '')])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src'})
parser.add_rule(picture_rule)
for data in request:
parser.feed(data.decode('utf-8'))
# print(picture_rule.get_result()[0]['src'])
return picture_rule.get_result()[0]['src']
def parse_index(self, request, url=URL()):
parser = SiteParser()
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('td', 'height', '500')])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src'})
picture_rule.set_attribute_modifier_function(
'src', lambda x: url.domain() + '/' + x)
parser.add_rule(picture_rule)
for data in request:
parser.feed(data.decode(encoding="utf-8", errors="ignore"))
return picture_rule.get_result()[0]['src']
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
domain = base_url.domain()
href_rule = ParserRule() # startpage & model's page
href_rule.add_activate_rule_level([('div', 'class', 'block galleries first'),
('div', 'class', 'block models'),
('div', 'class', 'block galleries')])
href_rule.add_activate_rule_level([('div', 'class', 'thumbs')])
href_rule.add_process_rule_level('a', {'href'})
href_rule.add_process_rule_level('img', {'src', 'alt'})
href_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
href_rule.set_attribute_modifier_function('src', lambda x: self.get_href(x, base_url))
parser.add_rule(href_rule)
href_model_page_rule = ParserRule() # page number in model's page
href_model_page_rule.add_activate_rule_level([('div', 'class', 'block galleries'),
('div', 'class', 'block models')])
href_model_page_rule.add_activate_rule_level([('ul', 'class', 'pagination')])
href_model_page_rule.add_process_rule_level('a', {'href'})
href_model_page_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(href_model_page_rule)
model_litera_rule = ParserRule()
model_litera_rule.add_activate_rule_level([('div', 'id', 'header')])
model_litera_rule.add_process_rule_level('a', {'href', 'title'})
model_litera_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(model_litera_rule)
picture_rule = ParserRule() # gallery rule
picture_rule.add_activate_rule_level([('div', 'class', 'block gallery')])
picture_rule.add_process_rule_level('a', {'href'})
picture_rule.add_process_rule_level('img', {'alt'})
picture_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(picture_rule)
picture_href_rule = ParserRule() # gallery href's rule
picture_href_rule.add_activate_rule_level([('div', 'class', 'profile')])
picture_href_rule.add_activate_rule_level([('div', 'class', 'cover')])
picture_href_rule.add_process_rule_level('a', {'href'})
picture_href_rule.add_process_rule_level('img', {'alt'})
picture_href_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(picture_href_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(href_rule.get_result()) > 0:
result.set_type('hrefs')
for item in href_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']), href=URL(item['href']), popup=item.get('alt', '')))
for item in model_litera_rule.get_result(['href', 'title', 'data']):
if item['title'].startswith('Met Art Models'):
result.add_control(ControlInfo(item['data'], URL(item['href'])))
for item in href_model_page_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
for f in picture_rule.get_result():
x = FullPictureInfo(abs_href=URL(f['href']), rel_name=f['href'].rpartition('/')[2])
result.add_full(x)
for f in picture_href_rule.get_result(['alt', 'href']):
# print(f)
result.add_control(ControlInfo(text=f['alt'], url=URL(f['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
site = None
for s in self.accepted_sites:
if base_url.contain(s['test']):
site = s
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([site['rule']])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('ul', 'class', 'pager'),
('div', 'class', 'navbartext'),
('div', 'class', 'navigation'),
('div', 'class', 'pager'),
('div', 'class', 'col-md-12 pager'),
('ul', 'id', 'pager')])
startpage_pages_rule.add_process_rule_level('a', {'href', 'alt'})
startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('article', 'class', 'b-margin-40 g148 gallery'),
('div', 'class', 'wrapper_g'),
('td', 'style', 'background:#ededed;'),
('div', 'id', 'gallerycont'),
('div', 'class', 'galleryblock'),
('div', 'class', 'list gallery'),
('div', 'class', 'picturecontainer mainpics'),
('div', 'class', 'single_thumb'),
('div', 'class', 'minithumbs')])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src'})
picture_rule.set_attribute_modifier_function('src', lambda text: text.replace('/tn_', '/').replace('_tn_', '_'))
parser.add_rule(picture_rule)
for s in open(fname, encoding='utf-8'):
parser.feed(s)
result = ParseResult()
if len(startpage_rule.get_result()) > 0:
# print('Startpage rule')
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']), href=URL(item['href']), popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
i = 1
for f in picture_rule.get_result(['src']):
# print(f)
result.add_full(FullPictureInfo(abs_href=URL(f['src']), rel_name='%03d.jpg' % i))
i += 1
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
# def star_get_url(txt=''):
# return txt.partition('(')[2].partition(')')[0]
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([
('div', 'class', 'list_videos'), ('div', 'class', 'list_albums'),
('div', 'class', 'list_videos model-girls-list'),
('div', 'class', 'list_videos list_channel')
])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
startpage_rule.set_attribute_modifier_function(
'src', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'pagination')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('div', 'class',
'model-alpha')])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href'})
# startpage_hrefs_rule.set_attribute_filter_function('href',lambda x: '/videos/' in x)
startpage_hrefs_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_hrefs_rule)
#
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'vids')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function(
'data', lambda text: 'video_url' in text)
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class',
'video-categories')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
gallery_user_rule = ParserRule(collect_data=True)
gallery_user_rule.add_activate_rule_level([('div', 'class',
'video-added-info')])
gallery_user_rule.add_process_rule_level('a', {'href'})
# gallery_user_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x+'/videos',base_url))
gallery_user_rule.set_attribute_filter_function(
'href', lambda x: '/members/' in x)
parser.add_rule(gallery_user_rule)
photo_rule = ParserRule()
photo_rule.add_activate_rule_level([('div', 'class', 'zoom-gallery')])
photo_rule.add_process_rule_level('a', {'href'})
# photo_rule.set_attribute_filter_function('href', lambda text: '/photos/' in text)
photo_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(photo_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
def add_href_and_user_to_result():
if gallery_user_rule.is_result(['href']):
for item in gallery_user_rule.get_result(['href']):
# print(item)
username = item['data'].strip().partition(
'Added by ')[2].partition(' ')[0]
# print(username)
if username != '':
result.add_control(
ControlInfo('"' + username + ' videos"',
URL(item['href'] + 'public_videos/')))
result.add_control(
ControlInfo('"' + username + ' photos"',
URL(item['href'] + 'albums/')))
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'], URL(f['href'])))
if video_rule.is_result(): # len(video_rule.get_result()) > 0:
# for item in video_rule.get_result():
# print('=============================')
# print(item['data'])
script = video_rule.get_result()[0]['data'].replace(' ', '')
# print(script)
url = script.partition("video_url:'")[2].partition("'")[0]
# print(url)
video = MediaData(URL(url))
result.set_type('video')
result.set_video(video)
add_href_and_user_to_result()
return result
if photo_rule.is_result():
result.set_type('pictures')
base_dir = base_url.get_path(base=Setting.base_dir) + base_url.get(
).rpartition('/')[2] + '/'
result.set_gallery_path(base_dir)
# print(base_dir)
for item in photo_rule.get_result():
name = item['href'].rpartition('/')[2].strip('*')
picture = FullPictureInfo(abs_href=URL(item['href']),
rel_name=name)
picture.set_base(base_dir)
result.add_full(picture)
add_href_and_user_to_result()
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result(['href']):
# print(item)
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(startpage_hrefs_rule.get_result(['href'])) > 0:
for item in startpage_hrefs_rule.get_result(['href', 'data']):
result.add_control(
ControlInfo(item.get('data', item.get('title', '')),
URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
def star_get_url(txt=''):
return txt.partition('(')[2].partition(')')[0]
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'thumb')])
startpage_rule.add_process_rule_level('a', {'title', 'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'wp-pagenavi')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('ul', 'class', 'list')])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href'})
startpage_hrefs_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_hrefs_rule)
#
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'id', 'video')])
video_rule.add_process_rule_level('iframe', {'src'})
# video_rule.set_attribute_filter_function('data', lambda text: 'flashvars' in text)
parser.add_rule(video_rule)
fake_video_rule = ParserRule()
fake_video_rule.add_activate_rule_level([('div', 'id', 'video')])
fake_video_rule.add_process_rule_level('div', {})
# video_rule.set_attribute_filter_function('data', lambda text: 'flashvars' in text)
parser.add_rule(fake_video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'id', 'extras')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result(): # len(video_rule.get_result()) > 0:
# for item in video_rule.get_result():
# print(item)
src = video_rule.get_result()[0]['src']
query = parse_qs(urlparse(video_rule.get_result()[0]['src'])[4])
alternates = list()
if 'f' in query:
data = {'data': query['f'][0]}
php_url = 'http://donfreeporn.com/wp-content/themes/detube/Htplugins/Loader.php*'
url = URL(php_url, 'POST', post_data=data)
r = load(url)
video_url = URL(r.json()['l'][0])
else:
r = load(URL(src))
setup = self.quotes(r.text, 'jwplayer("vplayer").setup(',
');').replace(' ', '')
sources = self.quotes(setup, 'sources:[{', '}],').split('},{')
for item in sources:
if '.mp4' in item:
# print(item)
file = self.quotes(item, 'file:"', '"')
label = self.quotes(item, 'label:"', '"')
# print(file,label)
alternates.append(dict(text=label,
url=URL(file + '*')))
if len(alternates) == 0:
return result
video_url = alternates[0]['url']
video = MediaData(video_url)
for item in alternates:
video.add_alternate(item)
result.set_type('video')
result.set_video(video)
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if fake_video_rule.is_result():
print('Broken video on this url')
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('title', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in startpage_hrefs_rule.get_result(['href']):
href = item['href']
label = href.split('/')[-2]
# print(label,href)
result.add_control(ControlInfo(label, URL(href)))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'link-3col')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
startpage_rule.set_attribute_modifier_function(
'src', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'wp-pagenavi')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
tags_rule = ParserRule()
tags_rule.add_activate_rule_level([('div', 'class', 'tagcloud')])
tags_rule.add_process_rule_level('a', {'href'})
tags_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(tags_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'player')])
video_rule.add_process_rule_level('iframe', {'src'})
video_rule.set_attribute_filter_function('src',
lambda x: 'fileone.tv' in x)
video_rule.set_attribute_modifier_function(
'src', lambda x: self.get_href(x, base_url))
parser.add_rule(video_rule)
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'id', 'extras')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result():
print(item)
try:
r = load(URL(item['src']))
setup = self.quotes(r.text, "jwplayer('player').setup(",
")").replace(' ', '')
file = self.quotes(setup, "file:'", "'")
urls.add("default", URL(file + '*'))
except LoaderError as err:
print(err)
result.set_video(urls.get_media_data())
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(
ControlInfo(f['data'].strip(), URL(f['href'])))
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
for item in startpage_rule.get_result(['href']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in tags_rule.get_result(['href', 'data']):
result.add_control(ControlInfo(item['data'],
URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([
('div', 'class', 'well well-sm hover'),
('div', 'class', 'channelContainer')
])
startpage_rule.add_process_rule_level('a', {'href', 'title'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
startpage_rule.set_attribute_modifier_function(
'src', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('ul', 'class',
'pagination')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('ul', 'class',
'drop2 hidden-xs')])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href'})
# startpage_hrefs_rule.set_attribute_filter_function('href',lambda x: '/videos/' in x)
startpage_hrefs_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_hrefs_rule)
#
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'video-container')
])
video_rule.add_process_rule_level('source', {'src', 'label', 'res'})
# video_rule.set_attribute_filter_function('data', lambda text: 'video_url' in text)
video_rule.set_attribute_modifier_function(
'src', lambda x: self.get_href(x, base_url))
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class',
'm-t-10 overflow-hidden')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
gallery_user_rule = ParserRule(collect_data=True)
gallery_user_rule.add_activate_rule_level([
('div', 'class', 'pull-left user-container')
])
gallery_user_rule.add_process_rule_level('a', {'href'})
gallery_user_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
gallery_user_rule.set_attribute_filter_function(
'href', lambda x: '#' not in x)
parser.add_rule(gallery_user_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result():
urls.add(item['res'], URL(item['src']))
result.set_video(urls.get_media_data(-1))
for f in gallery_user_rule.get_result(['data', 'href']):
result.add_control(
ControlInfo('"' + f['data'].strip() + '"', URL(f['href'])))
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if startpage_rule.is_result():
for item in startpage_rule.get_result(['href']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', item.get('title', ''))))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in startpage_hrefs_rule.get_result(['href', 'data']):
label = item['href'].strip('*').rpartition('/')[2]
result.add_control(ControlInfo(label, URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'item ')])
startpage_rule.add_process_rule_level('a', {'href', 'title'})
startpage_rule.add_process_rule_level('img', {'data-original', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
startpage_rule.set_attribute_modifier_function(
'src', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'pagination-holder')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'player')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function(
'data', lambda text: 'flashvars' in text)
# video_rule.set_attribute_modifier_function('src',lambda x:self.get_href(x,base_url))
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class', 'info')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result():
flashvars = self.quotes(
item['data'].replace(' ',
'').replace('\n',
'').replace('\t', ''),
'flashvars={', '};').split(',')
fv = dict()
for flashvar in flashvars:
split = flashvar.partition(':')
fv[split[0]] = split[2].strip("'\"")
files = dict()
for f in fv:
if fv[f].startswith('http://') and fv[f].rpartition(
'.')[2].strip('/') == 'mp4':
file = fv[f]
label = fv.get(f + '_text', f)
files[label] = file
for key in sorted(files.keys(), reverse=True):
urls.add(key, URL(files[key]))
result.set_video(urls.get_media_data())
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if startpage_rule.is_result():
for item in startpage_rule.get_result(['href']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['data-original']),
href=URL(item['href']),
popup=item.get('alt', item.get('title', ''))))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class',
'content-inner')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url) + '*')
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'pagination_link')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + x + '*')
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([
('div', 'class', 'sub_menu dark-menu'),
('div', 'class', 'sub-menu dark-menu')
])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href', 'title'})
startpage_hrefs_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url) + '*')
parser.add_rule(startpage_hrefs_rule)
#
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'id', 'content')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function(
'data', lambda text: 'jwplayer' in text)
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'id',
'media-tags-container')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + x + '*')
parser.add_rule(gallery_href_rule)
gallery_user_rule = ParserRule()
gallery_user_rule.add_activate_rule_level([('div', 'class',
'thumb-member-username')])
gallery_user_rule.add_process_rule_level('a', {'href'})
# gallery_user_rule.set_attribute_modifier_function('href', lambda x: base_url.domain() + x + '*')
# gallery_user_rule.set_attribute_filter_function('href',lambda x:'/categories/' in x)
parser.add_rule(gallery_user_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result():
file = self.quotes(item['data'].replace(' ', ''), '"file":"',
'"')
urls.add('default', URL(file + '*'))
result.set_video(urls.get_media_data())
if gallery_user_rule.is_result():
user = gallery_user_rule.get_result()[0]['href'].rpartition(
'/')[2]
result.add_control(
ControlInfo('"' + user + ' uploads"',
URL('http://motherless.com/u/' + user + '*')))
result.add_control(
ControlInfo(
'"' + user + ' gals"',
URL('http://motherless.com/galleries/member/' + user +
'*')))
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if startpage_rule.is_result():
for item in startpage_rule.get_result(['href']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in startpage_hrefs_rule.get_result(['href', 'data']):
result.add_control(
ControlInfo(item.get('title', item.get('data', '')),
URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
print(base_url.domain())
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'ownpost')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'nav')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + '/' + x + '*')
parser.add_rule(startpage_pages_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('div', 'class', 'gallery')])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src', 'class'})
picture_rule.set_attribute_modifier_function(
'src', lambda text: text.replace('/tn_', '/'))
parser.add_rule(picture_rule)
picture_href_rule = ParserRule()
picture_href_rule.add_activate_rule_level([('div', 'class', 'gallery')
])
picture_href_rule.add_process_rule_level('a', {'href'})
picture_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
picture_href_rule.set_attribute_filter_function(
'href', lambda x: x.find('/?category=') != -1)
parser.add_rule(picture_href_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(startpage_rule.get_result()) > 0:
# print('Startpage rule')
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href'] + '*'),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
i = 1
for f in picture_rule.get_result(['src', 'class']):
# print(f)
result.add_full(
FullPictureInfo(abs_href=URL(f['src']),
rel_name='%03d.jpg' % i))
i += 1
for f in picture_href_rule.get_result():
# print(f)
result.add_control(
ControlInfo(f['data'].replace(',', ''), URL(f['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
print(base_url.get(), base_url.domain())
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'lady')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + '/' + x)
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'nav_link')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + '/' + x)
parser.add_rule(startpage_pages_rule)
startpage_nav_rule = ParserRule()
startpage_nav_rule.add_activate_rule_level([('td', 'class', 'nav')])
startpage_nav_rule.add_process_rule_level('a', {'href'})
parser.add_rule(startpage_nav_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('td', 'align', 'center')])
picture_rule.add_process_rule_level('a', {'href'})
picture_rule.add_process_rule_level('img', {'src'})
picture_rule.set_attribute_modifier_function(
'src', lambda text: _del_thumb(text))
parser.add_rule(picture_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result(['href', 'src']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href'] + '*'),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(
ControlInfo(item['data'], URL(item['href'] + '*')))
for item in startpage_nav_rule.get_result(['href', 'data']):
if item['href'].startswith("http://www.deffki.su/"):
result.add_control(
ControlInfo(item['data'], URL(item['href'] + '*')))
if base_url.contain('?go=gal&id='):
result.set_type('pictures')
dirname = self.base_addr + base_url.get_path(
) + '/' + base_url.get().rpartition('=')[2] + '/'
result.set_gallery_path(dirname)
i = 1
for f in picture_rule.get_result(['src', 'href']):
if not f['href'].startswith('prv.php?id='): continue
if not f['src'].startswith('http://'): continue
# print(f['src'])
result.add_full(
FullPictureInfo(abs_href=URL(f['src']),
abs_name=dirname + '%03d.jpg' % i))
i += 1
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('ul', 'class', 'thumbs'),
('li', 'class', 'category')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img',
{'src', 'alt', 'data-original'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'pager paging')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level(
'span', {'data-query-key', 'data-query-value'})
# startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x,base_url))
parser.add_rule(startpage_pages_rule)
channels_rule = ParserRule()
channels_rule.add_activate_rule_level([('ul', 'class', 'tag-150-list')
])
channels_rule.add_process_rule_level('a', {'href'})
channels_rule.add_process_rule_level('img', {'src'})
channels_rule.set_attribute_filter_function(
'href', lambda x: '/channel/' in x or '/prime/' in x)
channels_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(channels_rule)
channel_categories_rule = ParserRule()
channel_categories_rule.add_activate_rule_level([
('ul', 'class', 'link-tag-list long-col')
])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
channel_categories_rule.add_process_rule_level(
'span', {'data-query-key', 'data-query-value'})
# startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x,base_url))
parser.add_rule(channel_categories_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class',
'player-container')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function(
'data', lambda text: 'players.push' in text)
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('ul', 'class',
'video-tag-list')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result(): # len(video_rule.get_result()) > 0:
script = video_rule.get_result()[0]['data'].replace(' ',
'').replace(
'\\', '')
sources = script.partition("'sources':{")[2].partition(
'}')[0].split(',')
urls = list()
for item in sources:
part = item.strip("\n\t'").partition("':'")
if part[2].startswith('http://'):
data = dict(text=part[0],
url=URL(part[2].strip("'") + '*'))
urls.append(data)
if len(urls) == 1:
video = MediaData(urls[0]['url'])
elif len(urls) > 1:
video = MediaData(urls[-1]['url'])
for item in urls:
video.add_alternate(item)
else:
return result
result.set_type('video')
result.set_video(video)
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(
ControlInfo(f['data'].strip(), URL(f['href'])))
return result
def add_key(old, key, value):
(addr, br, keys) = old.partition('?')
print(addr, br, keys)
pairs = keys.split('&')
print(pairs)
keys = ''
found = False
for pair in pairs:
if pair.startswith(key):
keys += key + '=' + value + '&'
found = True
else:
keys += pair + '&'
if not found:
keys += key + '=' + value
return addr + '?' + keys.strip('&')
def add_pages_info_to_result(rule, description_key='data-query-value'):
for item in rule.get_result(['data-query-key',
'data-query-value']):
print(item)
key = item['data-query-key']
val = item['data-query-value']
description = item[description_key].strip('\t')
old = base_url.get()
addr = add_key(old, key, val)
result.add_page(ControlInfo(description, URL(addr + '*')))
if channels_rule.is_result():
result.set_type('hrefs')
for item in channels_rule.get_result():
# print(item)
info = item['href'].rpartition('/')[2].strip('*')
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=info))
add_pages_info_to_result(channel_categories_rule,
description_key='data')
# for item in channel_categories_rule.get_result(['data-query-key', 'data-query-value']):
# print(item)
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result(['href']):
# print(item)
t_url = item.get('data-original', item['src'])
result.add_thumb(
ThumbInfo(thumb_url=URL(t_url),
href=URL(item['href']),
popup=item.get('alt', '')))
add_pages_info_to_result(startpage_pages_rule)
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
domain = base_url.domain()
href_rule = ParserRule() # startpage & model's page
href_rule.add_activate_rule_level([('div', 'class', 'galleries'),
('div', 'class', 'models'),
('div', 'class', 'videos')])
href_rule.add_activate_rule_level([('div', 'class', 'items')])
href_rule.add_process_rule_level('a', {'href'})
href_rule.add_process_rule_level('img', {'src', 'alt'})
href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
href_rule.set_attribute_modifier_function(
'src', lambda x: self.get_href(x, base_url))
parser.add_rule(href_rule)
href_page_rule = ParserRule() # page number in model's page
href_page_rule.add_activate_rule_level([('ul', 'class', 'pagination')])
href_page_rule.add_process_rule_level('a', {'href'})
href_page_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(href_page_rule)
model_litera_rule = ParserRule()
model_litera_rule.add_activate_rule_level([('span', 'class', 'chars')])
model_litera_rule.add_process_rule_level('a', {'href'})
model_litera_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(model_litera_rule)
picture_rule = ParserRule() # gallery rule
picture_rule.add_activate_rule_level([('div', 'class', 'picture')])
picture_rule.add_process_rule_level('a', {'href'})
picture_rule.add_process_rule_level('img', {'alt'})
picture_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(picture_rule)
video_rule = ParserRule() # gallery rule
video_rule.add_activate_rule_level([('div', 'class', 'video')])
video_rule.add_process_rule_level('source', {'src'})
# video_rule.add_process_rule_level('img', {'alt'})
# video_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x,base_url))
parser.add_rule(video_rule)
picture_href_rule = ParserRule() # gallery href's rule
picture_href_rule.add_activate_rule_level([('div', 'class', 'model')])
picture_href_rule.add_activate_rule_level([('div', 'class', 'links')])
picture_href_rule.add_process_rule_level('a', {'href'})
picture_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(picture_href_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(video_rule.get_result()) > 0:
result.set_video(
MediaData(URL(video_rule.get_result()[0]['src'] + '*')))
result.set_type('video')
for f in picture_href_rule.get_result(['href', 'data']):
# print(f)
result.add_control(
ControlInfo(text=f['data'], url=URL(f['href'])))
return result
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
for f in picture_rule.get_result():
x = FullPictureInfo(abs_href=URL(f['href']),
rel_name=f['href'].rpartition('/')[2])
result.add_full(x)
for f in picture_href_rule.get_result(['href', 'data']):
# print(f)
result.add_control(
ControlInfo(text=f['data'], url=URL(f['href'])))
return result
if len(href_rule.get_result()) > 0:
result.set_type('hrefs')
for item in href_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', '')))
for item in model_litera_rule.get_result(['href', 'data']):
result.add_control(ControlInfo(item['data'],
URL(item['href'])))
for item in href_page_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
def star_get_url(txt=''):
return txt.partition('(')[2].partition(')')[0]
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'videos_form')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'data-lazy-src'})
startpage_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'wp-pagenavi')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('ul', 'class', 'list')])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href'})
startpage_hrefs_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_hrefs_rule)
#
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'id', 'videos_page')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function('data', lambda text: 'jwplayer(' in text)
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'id', 'Categories')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result():
file = self.quotes(item['data'].replace(' ', ''), "file:'", "'")
urls.add('default', URL(file + '*'))
result.set_video(urls.get_media_data())
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result(['href']):
# print(item)
result.add_thumb(ThumbInfo(thumb_url=URL(item['data-lazy-src']), href=URL(item['href']),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in startpage_hrefs_rule.get_result(['href']):
href = item['href']
label = href.split('/')[-2]
# print(label,href)
result.add_control(ControlInfo(label, URL(href)))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
# def star_get_url(txt=''):
# return txt.partition('(')[2].partition(')')[0]
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'well wellov well-sm'),
('div', 'class', 'col-sm-4 m-t-15')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt', 'data-original'})
startpage_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('ul', 'class', 'pagination')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('div', 'class', 'btn-group')])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href'})
# startpage_hrefs_rule.set_attribute_filter_function('href',lambda x: '/videos/' in x)
startpage_hrefs_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_hrefs_rule)
#
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'container')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function('data', lambda text: 'nuevoplayer' in text)
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class', 'm-t-10 overflow-hidden catmenu')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
gallery_user_rule = ParserRule()
gallery_user_rule.add_activate_rule_level([('div', 'class', 'pull-left user-container')])
gallery_user_rule.add_process_rule_level('a', {'href'})
gallery_user_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x + '/videos', base_url))
gallery_user_rule.set_attribute_filter_function('href', lambda x: '/user/' in x)
parser.add_rule(gallery_user_rule)
photo_rule = ParserRule()
photo_rule.add_activate_rule_level([('div', 'class', 'panel-body')])
photo_rule.add_process_rule_level('a', {'href'})
photo_rule.set_attribute_filter_function('href', lambda text: '/photos/' in text)
photo_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(photo_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result(): # len(video_rule.get_result()) > 0:
script = video_rule.get_result()[0]['data']
urls = list()
for item in script.split(';'):
if 'var video' in item:
part = item.strip().partition('video_')[2].partition('=')
data = dict(text=part[0], url=URL(part[2].strip("'") + '*'))
urls.append(data)
if len(urls) == 1:
video = MediaData(urls[0]['url'])
elif len(urls) > 1:
video = MediaData(urls[-1]['url'])
for item in urls:
video.add_alternate(item)
else:
return result
result.set_type('video')
result.set_video(video)
#
# for f in gallery_channel_rule.get_result(['data', 'href']):
# result.add_control(ControlInfo(f['data'], URL(f['href'])))
if gallery_user_rule.is_result():
for item in gallery_user_rule.get_result():
# print(item)
# print('*'+item['data'].strip()+'*')
username = item['data'].strip()
if username != '':
result.add_control(ControlInfo('"' + username + '"', URL(item['href'])))
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if photo_rule.is_result():
result.set_type('pictures')
base_dir = base_url.get_path(base=Setting.base_dir)
result.set_gallery_path(base_dir)
for item in photo_rule.get_result():
name = item['href'].rpartition('/')[2].strip('*')
picture = FullPictureInfo(abs_href=URL(item['href']), rel_name=name)
picture.set_base(base_dir)
result.add_full(picture)
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result(['href']):
# print(item)
t_url = self.get_href(item.get('data-original', item['src']), base_url)
t_href = item['href']
# print(t_href)
if '/album/' in t_href:
# print(t_href)
t_href = t_href.replace('/album/', '/album/slideshow/')
# print(t_href)
result.add_thumb(ThumbInfo(thumb_url=URL(t_url), href=URL(t_href), popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(startpage_hrefs_rule.get_result(['href'])) > 0:
for item in startpage_hrefs_rule.get_result(['href', 'data']):
result.add_control(ControlInfo(item.get('title', item.get('data', '')), URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'thumb')])
startpage_rule.add_process_rule_level('a', {'href', 'title'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'wp-pagenavi')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
tags_rule = ParserRule()
tags_rule.add_activate_rule_level([('div', 'class', 'tagcloud')])
tags_rule.add_process_rule_level('a', {'href'})
tags_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(tags_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'videoContainer')])
video_rule.add_process_rule_level('iframe', {'src'})
video_rule.set_attribute_modifier_function('src', lambda x: self.get_href(x, base_url))
parser.add_rule(video_rule)
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'id', 'extras')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result():
try:
r = load(URL(item['src']))
r = load(URL(self.quotes(r.text, "jwplayer().load('", "'") + '*'))
source = self.quotes(r.text, '<item>', '</item>').strip()
split = source.split('<jwplayer:source file="')
for l in split:
if l is '':
continue
url = l.partition('"')[0]
label = self.quotes(l, 'label="', '"')
urls.add(label, URL(url + '*'))
except LoaderError as err:
print(err)
result.set_video(urls.get_media_data())
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'].strip(), URL(f['href'])))
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
for item in startpage_rule.get_result(['href']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']), href=URL(item['href']), popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in tags_rule.get_result(['href', 'data']):
result.add_control(ControlInfo(item['data'], URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([
('span', 'class', 'thumb_container_box short'),
('span', 'class', 'thumb_container_box long')
])
startpage_rule.add_process_rule_level('a', {'href', 'title'})
startpage_rule.add_process_rule_level('img', {'src'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'main-sectionpaging')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: util.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([
('ul', 'class', 'simple-list simple-list--channels')
])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href', 'title'})
# startpage_hrefs_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x,base_url) + '*')
parser.add_rule(startpage_hrefs_rule)
#
video_rule = ParserRule()
video_rule.add_activate_rule_level([('video', '', '')])
video_rule.add_process_rule_level('source', {'src', 'id'})
# video_rule.set_attribute_filter_function('data', lambda text: 'jwplayer' in text)
video_rule.set_attribute_modifier_function('src',
lambda txt: txt + '*')
parser.add_rule(video_rule)
video_script_rule = ParserRule()
video_script_rule.add_activate_rule_level([('body', '', '')])
video_script_rule.add_process_rule_level('script', {})
video_script_rule.set_attribute_filter_function(
'data', lambda text: 'shows:' in text)
# video_script_rule.set_attribute_modifier_function('src',lambda txt:txt+'*')
parser.add_rule(video_script_rule)
gallery_rule = ParserRule()
gallery_rule.add_activate_rule_level([('div', 'id', 'slideshow')])
gallery_rule.add_process_rule_level('a', {'index'})
gallery_rule.add_process_rule_level('img', {'src'})
# video_rule.set_attribute_filter_function('data', lambda text: 'jwplayer' in text)
gallery_rule.set_attribute_modifier_function(
'src', lambda txt: txt.replace('/thumbs/', '/'))
parser.add_rule(gallery_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class', 'added')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: util.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_script_rule.is_result() or video_rule.is_result():
files = set()
default_vid = None
for item in video_script_rule.get_result():
script = item['data'].replace(' ', '')
streams = up.unquote(self.quotes(script, '"streams":[', ']'))
while '"file":"' in streams:
split = streams.partition('"file":"')[2].partition('"')
streams = split[2]
files.add(split[0] + '*')
for item in video_rule.get_result():
files.add(item['src'])
if 'id' not in item:
default_vid = item['src']
if len(files) == 0:
return result
if default_vid is None:
default_vid = files.pop()
else:
files.discard(default_vid)
urls = UrlList()
urls.add('Default', URL(default_vid))
for item in files:
urls.add(item[-5:-1], URL(item))
result.set_video(urls.get_media_data())
for f in gallery_href_rule.get_result(['href']):
label = f['data'].strip()
result.add_control(ControlInfo(label, URL(f['href'])))
return result
if gallery_rule.is_result():
result.set_type('pictures')
url = URL(gallery_rule.get_result()[0]['src'] + '*')
base_dir = url.get_path(base=Setting.base_dir)
result.set_gallery_path(base_dir)
for f in gallery_rule.get_result():
fname = int(f['index'])
# print(fname)
picture = FullPictureInfo(
abs_href=URL(f['src'] + '*'),
rel_name='pic{0:03d}.jpg'.format(fname))
picture.set_base(base_dir)
result.add_full(picture)
for f in gallery_href_rule.get_result(['href']):
label = f['data'].strip()
result.add_control(ControlInfo(label, URL(f['href'])))
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
for item in startpage_rule.get_result(['href']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('title', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(startpage_hrefs_rule.get_result(['href'])) > 0:
for item in startpage_hrefs_rule.get_result(['href', 'data']):
result.add_control(
ControlInfo(item.get('title', item.get('data', '')),
URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
print(base_url.get(), base_url.domain())
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'post'),
('div', 'class', 'post300')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href',
lambda x: get_href(x))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'id', 'pager')])
startpage_pages_rule.add_activate_rule_level([('div', 'id', 'pc')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + x)
parser.add_rule(startpage_pages_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('div', 'id', 'cc')])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src', 'title'})
picture_rule.set_attribute_modifier_function(
'src', lambda text: _del_thumb(text))
parser.add_rule(picture_rule)
picture_href_rule = ParserRule()
picture_href_rule.add_activate_rule_level([('div', 'id', 'cc')])
picture_href_rule.add_activate_rule_level([('div', 'class',
'shorttext')])
picture_href_rule.add_process_rule_level('a', {'href', 'alt'})
parser.add_rule(picture_href_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result(['href', 'src']):
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href'] + '*'),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(
ControlInfo(item['data'], URL(item['href'] + '*')))
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
i = 1
for f in picture_rule.get_result(['src', 'title']):
result.add_full(
FullPictureInfo(abs_href=URL(f['src']),
rel_name='%03d.jpg' % i))
i += 1
for f in picture_href_rule.get_result():
if f['href'].startswith('/'):
result.add_control(
ControlInfo(text=f['alt'],
url=URL(base_url.domain() + f['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
# print(base_url.domain())
def star_get_url(txt=''):
return txt.partition('(')[2].partition(')')[0]
startpage_rule = ParserRule(debug=False)
startpage_rule.add_activate_rule_level([('ul', 'class', 'video-listing'),
('ul', 'class', 'video-listing two-in-row'),
('ul', 'class', 'video-listing four-in-row'),
('ul', 'class', 'video-listing two-in-row id-recommended-list'),
('ul', 'class', 'video-listing four-in-row id-recommended-list')
])
startpage_rule.add_process_rule_level('a', {'href', 'title'})
startpage_rule.add_process_rule_level('img', {'data-src', 'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
startpage_rule.set_attribute_modifier_function('data-src', lambda x: self.get_href(x, base_url))
startpage_rule.set_attribute_modifier_function('src', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'pageNumbersHolder')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('ul', 'class', 'categories-listing'),
('ul', 'class', 'categories-popular-listing'),
('ul', 'class', 'abc-categories newAbcCategories')])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_hrefs_rule.add_process_rule_level('a', {'href', 'title'})
startpage_hrefs_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_hrefs_rule)
pornstars_rule = ParserRule()
pornstars_rule.add_activate_rule_level([('div', 'id', 'all_pornstars')])
pornstars_rule.add_process_rule_level('a', {'href'})
pornstars_rule.add_process_rule_level('img', {'src', 'alt'})
# pornstars_rule.set_attribute_filter_function('href',lambda x: '/channel/' in x or '/prime/' in x)
pornstars_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(pornstars_rule)
pornstars_hrefs_rule = ParserRule()
pornstars_hrefs_rule.add_activate_rule_level([('ul', 'class', 'abc-categories newAbcCategories')])
# startpage_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
pornstars_hrefs_rule.add_process_rule_level('a', {'href'})
pornstars_hrefs_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(pornstars_hrefs_rule)
channels_rule = ParserRule(debug=False)
channels_rule.add_activate_rule_level([('ul', 'class', 'channels-list three-in-row')])
channels_rule.add_process_rule_level('a', {'href'})
channels_rule.add_process_rule_level('img', {'src', 'data-src', 'alt'})
channels_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
channels_rule.set_attribute_modifier_function('src', lambda x: self.get_href(x, base_url))
channels_rule.set_attribute_modifier_function('data-src', lambda x: self.get_href(x, base_url))
parser.add_rule(channels_rule)
channels_hrefs_rule = ParserRule()
channels_hrefs_rule.add_activate_rule_level([('div', 'class', 'channel-filters-categories')])
# channels_hrefs_rule.add_activate_rule_level([('a', 'class', 'current')])
channels_hrefs_rule.add_process_rule_level('a', {'href', 'title'})
channels_hrefs_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(channels_hrefs_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'watch')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function('data', lambda text: 'redtube_flv_player' in text)
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class', 'video-details')])
# gallery_href_rule.add_activate_rule_level([('td', 'class', 'links')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
gallery_href_rule.set_attribute_filter_function('href', lambda x: x != '*')
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result():
script = item['data'].replace(' ', '').replace('\\', '')
sources = self.quotes(script, 'sources:{"', '"},').split('","')
for f in sources:
t = f.partition('":"')
label = t[0]
file = self.get_href(t[2], base_url)
urls.add(label, URL(file))
result.set_video(urls.get_media_data())
for f in gallery_href_rule.get_result(['data', 'href']):
href = f['href']
label = f['data']
if '/redtube/' in href or '/tag/' in href:
result.add_control(ControlInfo(label, URL(href)))
elif '/pornstar/' in href:
ps_name = href.rstrip('*').rpartition('/')[2].replace('+', ' ').title()
result.add_control(ControlInfo(ps_name, URL(href)))
else:
# adding user
result.add_control(ControlInfo("'" + label + "'", URL(href.replace('*', '/videos*'))))
return result
if pornstars_rule.is_result():
result.set_caption_visible(True)
for item in pornstars_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']), href=URL(item['href']), popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in pornstars_hrefs_rule.get_result(['href']):
result.add_control(ControlInfo(item['data'], URL(item['href'])))
return result
if channels_rule.is_result():
result.set_caption_visible(True)
for item in channels_rule.get_result(['href']):
thumb_href = item.get('data-src', item.get('src'))
descr = item.get('alt', '').title()
result.add_thumb(ThumbInfo(thumb_url=URL(thumb_href), href=URL(item['href']), popup=descr))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in channels_hrefs_rule.get_result(['href']):
result.add_control(ControlInfo(item['title'], URL(item['href'])))
return result
if startpage_rule.is_result():
for item in startpage_rule.get_result(['href']):
thumb_href = item.get('data-src', item.get('src'))
descr = item.get('title', item.get('alt', ''))
result.add_thumb(ThumbInfo(thumb_url=URL(thumb_href), href=URL(item['href']), popup=descr))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in startpage_hrefs_rule.get_result(['href', 'title']):
result.add_control(ControlInfo(item['title'], URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'flower')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'morepartners')])
startpage_pages_rule.add_process_rule_level('a', {'href', 'alt'})
startpage_pages_rule.set_attribute_modifier_function('href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
picture_trigger_rule = ParserRule()
picture_trigger_rule.add_activate_rule_level([('a', 'class', 'thumbsmall')])
# picture_trigger_rule.add_process_rule_level('a', set())
picture_trigger_rule.add_process_rule_level('img', {'src'})
# picture_trigger_rule.set_attribute_modifier_function('src', lambda text: text.replace('/tn_', '/'))
parser.add_rule(picture_trigger_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('div', 'class', 'flower')])
picture_rule.add_process_rule_level('a', {'class'})
picture_rule.add_process_rule_level('img', {'src'})
picture_rule.set_attribute_modifier_function('src', lambda text: text.replace('/tn_', '/'))
parser.add_rule(picture_rule)
for s in open(fname, encoding='utf-8'):
parser.feed(s)
result = ParseResult()
if len(startpage_rule.get_result()) > 0:
# print('Startpage rule')
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']), href=URL(item['href']), popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(picture_trigger_rule.get_result()) > 0:
result.set_type('pictures')
i = 1
for f in picture_rule.get_result(['src', 'class']):
# print(f)
if f['class'] == 'thumbsmall':
result.add_full(FullPictureInfo(abs_href=URL(f['src']), rel_name='%03d.jpg' % i))
i += 1
return result
def parse_index_file(self, fname, base_url=URL()):
print('VP parsing')
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'bx'),
('div', 'class', 'bx lastrow')
])
startpage_rule.add_process_rule_level('a', {'href', 'class'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class',
'pagerwrap')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
# startpage_pages_rule.set_attribute_modifier_function('href',lambda x: base_url.domain() + x)
parser.add_rule(startpage_pages_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'video_panel')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function(
'data', lambda text: 'var flashvars' in text)
parser.add_rule(video_rule)
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class',
'tagas-secondrow')])
# gallery_href_rule.add_activate_rule_level([('td', 'class', 'btnList')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + x + '*')
# gallery_href_rule.set_attribute_filter_function('href',lambda x:'/category/'in x or '/search/'in x)
parser.add_rule(gallery_href_rule)
gallery_user_rule = ParserRule()
gallery_user_rule.add_activate_rule_level([('div', 'class', 'info')])
gallery_user_rule.add_process_rule_level('a', {'href'})
# gallery_user_rule.set_attribute_filter_function('href',lambda x:'/profile/' in x)
gallery_user_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x.replace('/user/', '/submitted/'),
base_url))
parser.add_rule(gallery_user_rule)
for s in open(fname, encoding='utf-8', errors='ignore'):
# print(s)
parser.feed(s.replace('</b>', '</a>'))
result = ParseResult()
if len(video_rule.get_result()) > 0:
script = video_rule.get_result()[0]['data'].replace(' ', '')
# print(script)
def get_url_from_script(script='', var=''):
data = script.partition('flashvars.' + var +
'="')[2].partition('"')[0]
# print(var,data)
if data.startswith('https://'): return URL(data)
videoUrlLow = get_url_from_script(script, 'videoUrlLow')
videoUrlLow2 = get_url_from_script(script, 'videoUrlLow2')
videoUrlMedium = get_url_from_script(script, 'videoUrlMedium')
videoUrlMedium2 = get_url_from_script(script, 'videoUrlMedium2')
videoUrlHD = get_url_from_script(script, 'videoUrlHD')
videoUrlHD2 = get_url_from_script(script, 'videoUrlHD2')
def add_alternate(video, txt, url):
if url is not None:
video.add_alternate(dict(text=txt, url=url))
# video=MediaData(videoUrlMedium)
if videoUrlMedium is not None:
video = MediaData(videoUrlMedium)
elif videoUrlLow is not None:
video = MediaData(videoUrlLow)
else:
print('No url found')
return result
add_alternate(video, 'Low', videoUrlLow)
add_alternate(video, 'Low2', videoUrlLow2)
add_alternate(video, 'Medium', videoUrlMedium)
add_alternate(video, 'Medium', videoUrlMedium2)
add_alternate(video, 'HD', videoUrlHD)
add_alternate(video, 'HD', videoUrlHD2)
result.set_type('video')
result.set_video(video)
for f in gallery_user_rule.get_result():
# print(f)
result.add_control(
ControlInfo('"' + f['data'] + '"', URL(f['href'])))
for f in gallery_href_rule.get_result(['data', 'href']):
# print(f)
result.add_control(ControlInfo(f['data'], URL(f['href'])))
# print('return')
return result
if len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item['alt']))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
# for item in startpage_hrefs_rule.get_result(['href', 'data']):
# result.add_control(ControlInfo(item['data'], URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class',
'fixed-content')])
startpage_rule.add_process_rule_level('a', {'href', 'class'})
startpage_rule.add_process_rule_level('div', {'style'})
startpage_rule.set_attribute_filter_function(
'class', lambda x: x == 'thumbnail')
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
startpage_rule.set_attribute_modifier_function(
'style', lambda x: x.partition("url('")[2].partition("')")[0])
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([
('div', 'class', 'col-xs-12 content-pagination')
])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
tags_rule = ParserRule()
tags_rule.add_activate_rule_level([('section', 'id', 'footer-tag')])
tags_rule.add_process_rule_level('a', {'href'})
tags_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(tags_rule)
categories_rule = ParserRule()
categories_rule.add_activate_rule_level([('ul', 'class',
'nav navbar-nav')])
categories_rule.add_process_rule_level('a', {'href'})
categories_rule.set_attribute_filter_function(
'href', lambda x: '/Category/' in x and "#" not in x)
categories_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(categories_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('body', '', '')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function(
'data', lambda text: 'angular.' in text)
parser.add_rule(video_rule)
#
gallery_href_rule = ParserRule()
gallery_href_rule.add_activate_rule_level([('div', 'class',
'row tag-area')])
gallery_href_rule.add_process_rule_level('a', {'href'})
gallery_href_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result(): # len(video_rule.get_result()) > 0:
script = video_rule.get_result()[0]['data'].replace(' ', '')
json_file_url = self.get_href(self.quotes(script, "host:'", "'"),
base_url)
# print(json_file_url)
from requests_loader import load, LoaderError
json_file = Setting.base_dir + 'tsp_video.json'
urls = list()
result.set_type('video')
try:
r = load(URL(json_file_url), json_file)
links = set()
for item in r.json()['mediaSources']:
# print(item)
if item['source'] not in links:
data = dict(text=item['quality'],
url=URL(item['source'] + '*'))
urls.append(data)
links.add(item['source'])
if len(urls) == 1:
video = MediaData(urls[0]['url'])
elif len(urls) > 1:
video = MediaData(urls[0]['url'])
for item in urls:
video.add_alternate(item)
else:
return result
result.set_video(video)
except LoaderError as err:
print(err)
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(
ControlInfo(f['data'].strip(), URL(f['href'])))
return result
if startpage_rule.is_result(): # len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result(['href']):
# print(item)
result.add_thumb(
ThumbInfo(thumb_url=URL(item['style']),
href=URL(item['href']),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
label = item['data'].replace(' ', '')
# print(item)
if len(label) > 0:
result.add_page(ControlInfo(label, URL(item['href'])))
if categories_rule.is_result(['href']):
for item in categories_rule.get_result(['href', 'data']):
result.add_control(
ControlInfo(item['data'], URL(item['href'])))
if tags_rule.is_result(['href']):
for item in tags_rule.get_result(['href', 'data']):
result.add_control(
ControlInfo(item['data'], URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'thumbs'),
('div', 'class', 'content_box domain2'),
('div', 'class', 'video_list'),
('div', 'class', 'video_list_models'),
('div', 'class', 'pics_list'),
('div', 'class', 'movie_thumbs')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', self.get_href)
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'pages'),
('div', 'class', 'pg')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function('href', lambda x: base_url.domain() + x)
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('li', 'class', 'orange dropdown')])
startpage_hrefs_rule.add_process_rule_level('a', {'href'})
startpage_hrefs_rule.set_attribute_filter_function('href', lambda txt: '/st/' in txt)
parser.add_rule(startpage_hrefs_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'player')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function('data', lambda text: 'video_url:' in text)
parser.add_rule(video_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([('div', 'class', 'thumb_box')])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src'})
picture_rule.set_attribute_modifier_function('src', lambda text: text.replace('t.jpg', '.jpg'))
parser.add_rule(picture_rule)
picture_href_rule = ParserRule()
picture_href_rule.add_activate_rule_level([('div', 'class', 'crumbles'),
('div', 'class', 'tags')])
picture_href_rule.add_process_rule_level('a', {'href', 'title'})
parser.add_rule(picture_href_rule)
for s in open(fname):
parser.feed(s)
result = ParseResult()
if video_rule.is_result():
video = MediaData(URL(self.get_attr_from_script(video_rule.get_result()[0]['data'])))
result.set_video(video)
result.set_type('video')
for f in picture_href_rule.get_result():
# print(f)
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if len(startpage_rule.get_result()) > 0:
# print('Startpage rule')
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']), href=URL(item['href']), popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
for item in startpage_hrefs_rule.get_result(['href', 'data']):
result.add_control(ControlInfo(item['data'], URL(item['href'])))
if len(picture_rule.get_result()) > 0:
result.set_type('pictures')
for f in picture_rule.get_result():
result.add_full(FullPictureInfo(rel_name=f['src']))
for f in picture_href_rule.get_result():
# print(f)
result.add_control(ControlInfo(f['title'], URL(f['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'thumbs'),
('div', 'class',
'movie_thumbs')])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function('href', get_href)
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'head')
])
startpage_pages_rule.add_activate_rule_level([('div', 'class', 'pages')
])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: 'http://www.tomorrowporn.com' + x)
parser.add_rule(startpage_pages_rule)
href_rule = ParserRule()
href_rule.add_activate_rule_level([('ul', 'class', 'sub_thumb_list')])
href_rule.add_process_rule_level('a', {'href'})
href_rule.add_process_rule_level('img', {'src', 'alt'})
href_rule.set_attribute_modifier_function('href', get_href)
parser.add_rule(href_rule)
picture_rule = ParserRule()
picture_rule.add_activate_rule_level([
('div', 'class', 'thumb_box'),
('div', 'class', 'thumb_box bottom_corners'),
('div', 'class', 'thumb_box top_corners')
])
picture_rule.add_process_rule_level('a', set())
picture_rule.add_process_rule_level('img', {'src'})
picture_rule.set_attribute_modifier_function(
'src', lambda text: text.replace('t', ''))
parser.add_rule(picture_rule)
picture_href_rule = ParserRule()
picture_href_rule.add_activate_rule_level([('div', 'class', 'menus')])
picture_href_rule.add_process_rule_level('h2', set())
picture_href_rule.add_process_rule_level('a', {'href', 'title'})
parser.add_rule(picture_href_rule)
for s in open(fname, encoding='utf-8'):
parser.feed(s)
result = ParseResult()
if len(startpage_rule.get_result()) > 0:
# result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
if len(href_rule.get_result()) > 0:
# result.set_type('hrefs')
for item in href_rule.get_result():
# print (item)
if 'src' in item:
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href']),
popup=item.get('alt', '')))
if len(picture_rule.get_result()) > 0:
# result.set_type('pictures')
for f in picture_rule.get_result():
result.add_full(FullPictureInfo(rel_name=f['src']))
for f in picture_href_rule.get_result():
# print(f)
result.add_control(ControlInfo(f['title'], URL(f['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'block-video')
])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src'})
# startpage_rule.set_attribute_modifier_function('href', lambda x: base_url.domain() + x)
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('ul', 'class',
'pagination')])
startpage_pages_rule.add_process_rule_level('a', {'title', 'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: base_url.domain() + x)
parser.add_rule(startpage_pages_rule)
startpage_hrefs_rule = ParserRule()
startpage_hrefs_rule.add_activate_rule_level([('li', 'id',
'categories')])
startpage_hrefs_rule.add_process_rule_level('a', {'href'})
# startpage_hrefs_rule.set_attribute_modifier_function('href', lambda x: base_url.domain() + x)
parser.add_rule(startpage_hrefs_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('div', 'class', 'player_body')])
video_rule.add_process_rule_level('script', {})
video_rule.set_attribute_filter_function(
'data', lambda text: 'function playStart()' in text)
parser.add_rule(video_rule)
picture_href_rule = ParserRule()
picture_href_rule.add_activate_rule_level([
('p', 'class', 'info_category'), ('p', 'class', 'info_tags'),
('p', 'class', 'info_cast')
])
picture_href_rule.add_process_rule_level('a', {'href'})
parser.add_rule(picture_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if len(video_rule.get_result()) > 0:
result.set_video(
MediaData(
URL(
self.get_attr_from_script(
video_rule.get_result()[0]['data']))))
result.set_type('video')
for f in picture_href_rule.get_result():
# print(f)
result.add_control(ControlInfo(f['data'], URL(f['href'])))
return result
if len(startpage_rule.get_result()) > 0:
result.set_type('hrefs')
for item in startpage_rule.get_result():
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(item['href'])))
for item in startpage_pages_rule.get_result(['href', 'data']):
if '?from=' in item['href']:
result.add_page(
ControlInfo(item['data'], URL(item['href'] + '*')))
else:
result.add_page(
ControlInfo(item['data'], URL(item['href'])))
for item in startpage_hrefs_rule.get_result(['href', 'data']):
result.add_control(ControlInfo(item['data'],
URL(item['href'])))
return result
def parse_index_file(self, fname, base_url=URL()):
parser = SiteParser()
startpage_rule = ParserRule()
startpage_rule.add_activate_rule_level([('div', 'class', 'video-thumb')
])
startpage_rule.add_process_rule_level('a', {'href'})
startpage_rule.add_process_rule_level('img', {'src', 'alt'})
startpage_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_rule)
startpage_pages_rule = ParserRule()
startpage_pages_rule.add_activate_rule_level([('ul', 'class',
'pagination')])
# startpage_pages_rule.add_activate_rule_level([('a', 'class', 'current')])
startpage_pages_rule.add_process_rule_level('a', {'href'})
startpage_pages_rule.set_attribute_modifier_function(
'href', lambda x: self.get_href(x, base_url))
parser.add_rule(startpage_pages_rule)
video_rule = ParserRule()
video_rule.add_activate_rule_level([('video', '', '')])
video_rule.add_process_rule_level('source', {'src', 'label', 'res'})
# video_rule.set_attribute_filter_function('data', lambda text: 'jwplayer' in text)
parser.add_rule(video_rule)
gallery_href_rule = ParserRule()
# gallery_href_rule.add_activate_rule_level([('div', 'class', 'option')])
gallery_href_rule.add_activate_rule_level([('div', 'class',
'tags-container')])
gallery_href_rule.add_process_rule_level('a', {'href'})
# gallery_href_rule.set_attribute_modifier_function('href', lambda x: base_url.domain() + x + '*')
parser.add_rule(gallery_href_rule)
self.proceed_parcing(parser, fname)
result = ParseResult()
if video_rule.is_result():
urls = UrlList()
for item in video_rule.get_result(['src', 'res']):
urls.add(item['res'], URL(item['src']))
result.set_video(urls.get_media_data(-1))
for f in gallery_href_rule.get_result(['data', 'href']):
result.add_control(ControlInfo(f['data'],
URL(f['href'] + '*')))
return result
if startpage_rule.is_result():
# for item in startpage_rule.get_result():
# print(item)
for item in startpage_rule.get_result(['href', 'src']):
href = item['href']
if '/category/' in href:
result.set_caption_visible(True)
result.add_thumb(
ThumbInfo(thumb_url=URL(item['src']),
href=URL(href),
popup=item.get('alt', '')))
for item in startpage_pages_rule.get_result(['href', 'data']):
result.add_page(ControlInfo(item['data'], URL(item['href'])))
return result